diff --git a/examples/simple-usage.nix b/examples/simple-usage.nix new file mode 100644 index 0000000..e69de29 diff --git a/mach_nix/data/data_interface.py b/mach_nix/data/data_interface.py index 8b82534..34346db 100644 --- a/mach_nix/data/data_interface.py +++ b/mach_nix/data/data_interface.py @@ -145,15 +145,15 @@ class NixpkgsDirectory(UserDict): return False return ver1.release[ver_idx] == ver2.release[ver_idx] - def choose_nix_pkg_for_py_pkg(self, name, ver): + def find_best_nixpkgs_candidate(self, name, ver): """ - In case a python package has more than one definition in nixpkgs + In case a python package has more than one candidate in nixpkgs like `django` and `django_2_2`, this algo will select the right one. """ pkgs: List[NixpkgsPyPkg] = sorted(self.get_all_candidates(name), key=lambda pkg: pkg.ver) if len(pkgs) == 1: return self[name].nix_key - # try to find nixpkgs pkg with best matching version + # try to find nixpkgs candidate with closest version remaining_pkgs = pkgs for i in range(7): # usually there are not more than 4 parts in a version same_ver = list(filter(lambda p: self.is_same_ver(ver, p.ver, i), remaining_pkgs)) diff --git a/mach_nix/ensure_nix.py b/mach_nix/ensure_nix.py new file mode 100644 index 0000000..9f89e5f --- /dev/null +++ b/mach_nix/ensure_nix.py @@ -0,0 +1,38 @@ +import os +import subprocess as sp +import sys +import urllib.request + + +def is_nix_available(): + nix_installed = False + try: + sp.run(['nix', '--version'], check=True, capture_output=True) + nix_installed = True + except FileNotFoundError: + pass + return nix_installed + + +def ensure_nix(): + if is_nix_available(): + return + print("The nix package manager is required! Install it now? [Y/n]: ", end='') + try: + answer = input() + if not answer or answer[0].lower() != 'y': + exit(1) + except KeyboardInterrupt: + exit(1) + with urllib.request.urlopen('https://nixos.org/nix/install') as f: + install_script = f.read() + read, write = os.pipe() + os.write(write, install_script) + os.close(write) + proc = sp.run('bash', stdin=read) + if proc.returncode: + print("Error while installing nix. Please check https://nixos.org/download.html for manual installation.", + file=sys.stderr) + exit(1) + print('Please activate nix like described above, then re-run mach-nix.') + exit(0) diff --git a/mach_nix/generators/overlay_generator.py b/mach_nix/generators/overlay_generator.py index 994d4c0..3c154a4 100644 --- a/mach_nix/generators/overlay_generator.py +++ b/mach_nix/generators/overlay_generator.py @@ -135,7 +135,7 @@ class OverlaysGenerator(ExpressionGenerator): # generate package overlays either via `overrideAttrs` if package already exists in nixpkgs, # or by creating it from scratch using `buildPythonPackage` if self.nixpkgs.exists(pkg.name): - nix_name = self.nixpkgs.choose_nix_pkg_for_py_pkg(pkg.name, pkg.ver) + nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver) out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str) master_key = self._get_ref_name(pkg.name, pkg.ver) other_names = (p.nix_key for p in self.nixpkgs.get_all_candidates(pkg.name) if p.nix_key != master_key) @@ -151,7 +151,7 @@ class OverlaysGenerator(ExpressionGenerator): def _get_ref_name(self, name, ver) -> str: if self.nixpkgs.exists(name): - return self.nixpkgs.choose_nix_pkg_for_py_pkg(name, ver) + return self.nixpkgs.find_best_nixpkgs_candidate(name, ver) return name def _needs_overlay(self, name, ver): diff --git a/mach_nix/nix/expression.nix b/mach_nix/nix/expression.nix index dc93247..81df0ea 100644 --- a/mach_nix/nix/expression.nix +++ b/mach_nix/nix/expression.nix @@ -1,29 +1,29 @@ { - requirements, - nixpkgs_src ? (import ./nixpkgs-src.nix).stable, # to take python packages from + requirements, # content from a requirements.txt file python_attr ? "python3", # python attr name inside given nixpkgs. Used as base for resulting python environment - prefer_nixpkgs ? true, # Prefer python package versions from nixpkgs instead of newer ones. Decreases build time. - disable_checks ? true, # Disable tests wherever possible to decrease build time. - nixpkgs_commit ? builtins.readFile ./NIXPKGS_COMMIT, - nixpkgs_tarball_sha256 ? builtins.readFile ./NIXPKGS_TARBALL_SHA256, - pypi_deps_db_commit ? builtins.readFile ./PYPI_DEPS_DB_COMMIT, + prefer_nixpkgs ? true, # Prefer python package versions from nixpkgs instead of newer ones. Decreases build time. + disable_checks ? true, # Disable tests wherever possible to decrease build time. + nixpkgs_commit ? builtins.readFile ./NIXPKGS_COMMIT, # nixpkgs version to use python packages from + nixpkgs_tarball_sha256 ? builtins.readFile ./NIXPKGS_TARBALL_SHA256, # nixpkgs version to use python packages from + pypi_deps_db_commit ? builtins.readFile ./PYPI_DEPS_DB_COMMIT, # python dependency DB version to use # Hash obtained using `nix-prefetch-url --unpack https://github.com/DavHau/pypi-deps-db/tarball/.tar.gz` - pypi_deps_db_sha256 ? builtins.readFile ./PYPI_DEPS_DB_TARBALL_SHA256 + pypi_deps_db_sha256 ? builtins.readFile ./PYPI_DEPS_DB_TARBALL_SHA256 # python dependency DB version to use }: let - nixpkgs_src = builtins.fetchTarball { + nixpkgs_src = (import ./nixpkgs-src.nix).stable; + pkgs = import nixpkgs_src { config = {}; }; + target_nixpkgs_src = builtins.fetchTarball { name = "nixpkgs"; url = "https://github.com/nixos/nixpkgs/tarball/${nixpkgs_commit}"; sha256 = "${nixpkgs_tarball_sha256}"; }; - pkgs = import nixpkgs_src { config = {}; }; - python = pkgs."${python_attr}"; - nixpkgs_json = import ./nixpkgs-json.nix { inherit pkgs python; }; - builder_pkgs = import (import ./nixpkgs-src.nix).stable { config = {}; }; - app = import ../../default.nix; + target_pkgs = import target_nixpkgs_src { config = {}; }; + target_python = target_pkgs."${python_attr}"; + nixpkgs_json = import ./nixpkgs-json.nix { pkgs = target_pkgs; python = target_python; }; builder_python = pkgs.python37.withPackages(ps: - (pkgs.lib.attrValues (pkgs.callPackage ./python-deps.nix {}) ++ [ app ]) + (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python37; fetchurl = pkgs.fetchurl; })) ); + src = ./../../.; pypi_deps_db_src = builtins.fetchTarball { name = "pypi-deps-db-src"; url = "https://github.com/DavHau/pypi-deps-db/tarball/${pypi_deps_db_commit}"; @@ -37,17 +37,17 @@ let sha256 = "${pypi_fetcher_tarball_sha256}"; }; expression = pkgs.runCommand "python-expression" - { buildInputs = [ builder_python pypi_deps_db_src]; + { buildInputs = [ src builder_python pypi_deps_db_src]; inherit disable_checks nixpkgs_commit nixpkgs_tarball_sha256 nixpkgs_json prefer_nixpkgs requirements pypi_fetcher_commit pypi_fetcher_tarball_sha256; - py_ver_str = python.version; + py_ver_str = target_python.version; pypi_deps_db_data_dir = "${pypi_deps_db_src}/data"; } '' mkdir -p $out/share export out_file=$out/share/expr.nix - export PYTHONPATH=${app}/lib/python3.7/site-packages/ - ${builder_python}/bin/python ${builder_python}/lib/python3.7/site-packages/mach_nix/generate.py + export PYTHONPATH=${src} + ${builder_python}/bin/python ${src}/mach_nix/generate.py ''; in -expression \ No newline at end of file +expression diff --git a/mach_nix/run.py b/mach_nix/run.py index 5e5cb4a..411fd99 100644 --- a/mach_nix/run.py +++ b/mach_nix/run.py @@ -3,38 +3,13 @@ import os import subprocess as sp import sys import tempfile -import urllib from argparse import ArgumentParser from os.path import realpath, dirname -from urllib.request import urlretrieve +from mach_nix.ensure_nix import ensure_nix from mach_nix.versions import PyVer -def ensure_nix(): - nix_installed = True - try: - sp.run(['nix', '--version'], check=True, capture_output=True) - except FileNotFoundError: - nix_installed = False - if nix_installed: - return - print("The nix package manager is required! Install it now? [Y/n]: ", end='') - answer = input() - if not answer or answer[0].lower() != 'y': - exit(1) - with urllib.request.urlopen('https://nixos.org/nix/install') as f: - install_script = f.read() - read, write = os.pipe() - os.write(write, install_script) - os.close(write) - proc = sp.run('sh', stdin=read) - if proc.returncode: - print("Error while installing nix. Please check https://nixos.org/download.html and install manually.", - file=sys.stderr) - exit(1) - - def gen(args, quiet: bool, return_expr=False): with open(args.r) as f: requirements = f.read().strip() @@ -79,7 +54,13 @@ def env(args): python.write(expr) shell.write("(import ./python.nix).env\n") default.write("import ./shell.nix\n") - print(f"created files: {python_nix_file}, {shell_nix_file}") + print(f"\nInitialized python environment in {target_dir}\n" + f"To activate it, execute: 'nix-shell {target_dir}'") + + +def be_patient(): + print("Generating python environment... If you run this the first time, the python package index " + "and dependency graph (~200MB) need to be downloaded. Please stay patient!") def main(): @@ -101,6 +82,7 @@ def main(): args = parser.parse_args() ensure_nix() + be_patient() if args.command == 'gen': gen(args, quiet=not args.o) diff --git a/setup.py b/setup.py index 7eafab8..9e7614c 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from os.path import abspath, dirname from setuptools import setup, find_packages pwd = dirname(abspath(__file__)) -with open(pwd + '/' + 'VERSION') as f: +with open(pwd + '/VERSION') as f: version = f.read().strip() setup( @@ -19,6 +19,8 @@ setup( "mach-nix = mach_nix:main" ], }, + package_data={'': ['nix/*']}, + include_package_data=True, install_requires=[ 'distlib ~= 0.3.0', 'packaging >= 19.0', diff --git a/shell.nix b/shell.nix index 672b2c4..590b227 100644 --- a/shell.nix +++ b/shell.nix @@ -6,4 +6,4 @@ mkShell { shellHook = '' export PYTHONPATH=$(pwd)/ ''; -} \ No newline at end of file +}