improve installation for other platforms

mach_nix/data/data_interface.py

@@ -145,15 +145,15 @@ class NixpkgsDirectory(UserDict):
             return False
         return ver1.release[ver_idx] == ver2.release[ver_idx]
 
-    def choose_nix_pkg_for_py_pkg(self, name, ver):
+    def find_best_nixpkgs_candidate(self, name, ver):
         """
-        In case a python package has more than one definition in nixpkgs
+        In case a python package has more than one candidate in nixpkgs
         like `django` and `django_2_2`, this algo will select the right one.
         """
         pkgs: List[NixpkgsPyPkg] = sorted(self.get_all_candidates(name), key=lambda pkg: pkg.ver)
         if len(pkgs) == 1:
             return self[name].nix_key
-        # try to find nixpkgs pkg with best matching version
+        # try to find nixpkgs candidate with closest version
         remaining_pkgs = pkgs
         for i in range(7):  # usually there are not more than 4 parts in a version
             same_ver = list(filter(lambda p: self.is_same_ver(ver, p.ver, i), remaining_pkgs))

mach_nix/ensure_nix.py

@@ -0,0 +1,38 @@
+import os
+import subprocess as sp
+import sys
+import urllib.request
+
+
+def is_nix_available():
+    nix_installed = False
+    try:
+        sp.run(['nix', '--version'], check=True, capture_output=True)
+        nix_installed = True
+    except FileNotFoundError:
+        pass
+    return nix_installed
+
+
+def ensure_nix():
+    if is_nix_available():
+        return
+    print("The nix package manager is required! Install it now? [Y/n]: ", end='')
+    try:
+        answer = input()
+        if not answer or answer[0].lower() != 'y':
+            exit(1)
+    except KeyboardInterrupt:
+        exit(1)
+    with urllib.request.urlopen('https://nixos.org/nix/install') as f:
+        install_script = f.read()
+    read, write = os.pipe()
+    os.write(write, install_script)
+    os.close(write)
+    proc = sp.run('bash', stdin=read)
+    if proc.returncode:
+        print("Error while installing nix. Please check https://nixos.org/download.html for manual installation.",
+              file=sys.stderr)
+        exit(1)
+    print('Please activate nix like described above, then re-run mach-nix.')
+    exit(0)

mach_nix/generators/overlay_generator.py

@@ -135,7 +135,7 @@ class OverlaysGenerator(ExpressionGenerator):
             # generate package overlays either via `overrideAttrs` if package already exists in nixpkgs,
             # or by creating it from scratch using `buildPythonPackage`
             if self.nixpkgs.exists(pkg.name):
-                nix_name = self.nixpkgs.choose_nix_pkg_for_py_pkg(pkg.name, pkg.ver)
+                nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver)
                 out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
                 master_key = self._get_ref_name(pkg.name, pkg.ver)
                 other_names = (p.nix_key for p in self.nixpkgs.get_all_candidates(pkg.name) if p.nix_key != master_key)
@@ -151,7 +151,7 @@ class OverlaysGenerator(ExpressionGenerator):
 
     def _get_ref_name(self, name, ver) -> str:
         if self.nixpkgs.exists(name):
-            return self.nixpkgs.choose_nix_pkg_for_py_pkg(name, ver)
+            return self.nixpkgs.find_best_nixpkgs_candidate(name, ver)
         return name
 
     def _needs_overlay(self, name, ver):

mach_nix/nix/expression.nix

@@ -1,29 +1,29 @@
 {
-  requirements,
+  requirements,  # content from a requirements.txt file
-  nixpkgs_src ? (import ./nixpkgs-src.nix).stable,  # to take python packages from
   python_attr ? "python3",  # python attr name inside given nixpkgs. Used as base for resulting python environment
-  prefer_nixpkgs ? true, # Prefer python package versions from nixpkgs instead of newer ones. Decreases build time.
+  prefer_nixpkgs ? true,  # Prefer python package versions from nixpkgs instead of newer ones. Decreases build time.
-  disable_checks ? true, # Disable tests wherever possible to decrease build time.
+  disable_checks ? true,  # Disable tests wherever possible to decrease build time.
-  nixpkgs_commit ? builtins.readFile ./NIXPKGS_COMMIT,
+  nixpkgs_commit ? builtins.readFile ./NIXPKGS_COMMIT,  # nixpkgs version to use python packages from
-  nixpkgs_tarball_sha256 ? builtins.readFile ./NIXPKGS_TARBALL_SHA256,
+  nixpkgs_tarball_sha256 ? builtins.readFile ./NIXPKGS_TARBALL_SHA256,  # nixpkgs version to use python packages from
-  pypi_deps_db_commit ? builtins.readFile ./PYPI_DEPS_DB_COMMIT,
+  pypi_deps_db_commit ? builtins.readFile ./PYPI_DEPS_DB_COMMIT,  # python dependency DB version to use
   # Hash obtained using `nix-prefetch-url --unpack https://github.com/DavHau/pypi-deps-db/tarball/<pypi_deps_db_commit>.tar.gz`
-  pypi_deps_db_sha256 ? builtins.readFile ./PYPI_DEPS_DB_TARBALL_SHA256
+  pypi_deps_db_sha256 ? builtins.readFile ./PYPI_DEPS_DB_TARBALL_SHA256  # python dependency DB version to use
 }:
 let
-  nixpkgs_src = builtins.fetchTarball {
+  nixpkgs_src = (import ./nixpkgs-src.nix).stable;
+  pkgs = import nixpkgs_src { config = {}; };
+  target_nixpkgs_src = builtins.fetchTarball {
     name = "nixpkgs";
     url = "https://github.com/nixos/nixpkgs/tarball/${nixpkgs_commit}";
     sha256 = "${nixpkgs_tarball_sha256}";
   };
-  pkgs = import nixpkgs_src { config = {}; };
+  target_pkgs = import target_nixpkgs_src { config = {}; };
-  python = pkgs."${python_attr}";
+  target_python = target_pkgs."${python_attr}";
-  nixpkgs_json = import ./nixpkgs-json.nix { inherit pkgs python; };
+  nixpkgs_json = import ./nixpkgs-json.nix { pkgs = target_pkgs; python = target_python; };
-  builder_pkgs = import (import ./nixpkgs-src.nix).stable { config = {}; };
-  app = import ../../default.nix;
   builder_python = pkgs.python37.withPackages(ps:
-    (pkgs.lib.attrValues (pkgs.callPackage ./python-deps.nix {}) ++ [ app ])
+    (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python37; fetchurl = pkgs.fetchurl; }))
   );
+  src = ./../../.;
   pypi_deps_db_src = builtins.fetchTarball {
     name = "pypi-deps-db-src";
     url = "https://github.com/DavHau/pypi-deps-db/tarball/${pypi_deps_db_commit}";
@@ -37,17 +37,17 @@ let
     sha256 = "${pypi_fetcher_tarball_sha256}";
   };
   expression = pkgs.runCommand "python-expression"
-    { buildInputs = [ builder_python pypi_deps_db_src];
+    { buildInputs = [ src builder_python pypi_deps_db_src];
       inherit disable_checks nixpkgs_commit nixpkgs_tarball_sha256 nixpkgs_json
               prefer_nixpkgs requirements pypi_fetcher_commit pypi_fetcher_tarball_sha256;
-      py_ver_str = python.version;
+      py_ver_str = target_python.version;
       pypi_deps_db_data_dir = "${pypi_deps_db_src}/data";
     }
     ''
       mkdir -p $out/share
       export out_file=$out/share/expr.nix
-      export PYTHONPATH=${app}/lib/python3.7/site-packages/
+      export PYTHONPATH=${src}
-      ${builder_python}/bin/python ${builder_python}/lib/python3.7/site-packages/mach_nix/generate.py
+      ${builder_python}/bin/python ${src}/mach_nix/generate.py
     '';
 in
-expression
+expression

mach_nix/run.py

@@ -3,38 +3,13 @@ import os
 import subprocess as sp
 import sys
 import tempfile
-import urllib
 from argparse import ArgumentParser
 from os.path import realpath, dirname
-from urllib.request import urlretrieve
 
+from mach_nix.ensure_nix import ensure_nix
 from mach_nix.versions import PyVer
 
 
-def ensure_nix():
-    nix_installed = True
-    try:
-        sp.run(['nix', '--version'], check=True, capture_output=True)
-    except FileNotFoundError:
-        nix_installed = False
-    if nix_installed:
-        return
-    print("The nix package manager is required! Install it now? [Y/n]: ", end='')
-    answer = input()
-    if not answer or answer[0].lower() != 'y':
-        exit(1)
-    with urllib.request.urlopen('https://nixos.org/nix/install') as f:
-        install_script = f.read()
-    read, write = os.pipe()
-    os.write(write, install_script)
-    os.close(write)
-    proc = sp.run('sh', stdin=read)
-    if proc.returncode:
-        print("Error while installing nix. Please check https://nixos.org/download.html and install manually.",
-              file=sys.stderr)
-        exit(1)
-
-
 def gen(args, quiet: bool, return_expr=False):
     with open(args.r) as f:
         requirements = f.read().strip()
@@ -79,7 +54,13 @@ def env(args):
                 python.write(expr)
                 shell.write("(import ./python.nix).env\n")
                 default.write("import ./shell.nix\n")
-    print(f"created files: {python_nix_file}, {shell_nix_file}")
+    print(f"\nInitialized python environment in {target_dir}\n"
+          f"To activate it, execute: 'nix-shell {target_dir}'")
+
+
+def be_patient():
+    print("Generating python environment... If you run this the first time, the python package index "
+          "and dependency graph (~200MB) need to be downloaded. Please stay patient!")
 
 
 def main():
@@ -101,6 +82,7 @@ def main():
     args = parser.parse_args()
 
     ensure_nix()
+    be_patient()
 
     if args.command == 'gen':
         gen(args, quiet=not args.o)

setup.py

@@ -3,7 +3,7 @@ from os.path import abspath, dirname
 from setuptools import setup, find_packages
 
 pwd = dirname(abspath(__file__))
-with open(pwd + '/' + 'VERSION') as f:
+with open(pwd + '/VERSION') as f:
     version = f.read().strip()
 
 setup(
@@ -19,6 +19,8 @@ setup(
             "mach-nix = mach_nix:main"
         ],
     },
+    package_data={'': ['nix/*']},
+    include_package_data=True,
     install_requires=[
         'distlib ~= 0.3.0',
         'packaging >= 19.0',

shell.nix

@@ -6,4 +6,4 @@ mkShell {
   shellHook = ''
     export PYTHONPATH=$(pwd)/
   '';
-}
+}