20 KiB
Composable shell.nix
- Introduction
nix-shell-fu
level 1 lessonnix-shell-fu
level 2 lesson; scripting and configuringnix-shell-fu
level 3 lesson; composability- Appendix
- tags
- nix
TL;DR: This is how I created a docker-compose
replacement with nix-shell
.
Here is a solution to have a composable nix shell representation focused on
replacing docker-compose
.
Introduction
At work we use docker-compose
to run integration tests on a big project that need
to connect to multiple different databases as well as a few other services.
This article is about how to replace docker-compose
by nix
for a local dev
environment.
nix-shell-fu
level 1 lesson
Let's start with a basic shell.nix
example:
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz) {} }:
with pkgs: mkShell
{ buildInputs = [ hello ];
shellHook = ''
echo "Using ${hello.name}."
'';
}
And this could be understood in plain English as:
In the packages of nix version 22.11, create a new shell into which the package
hello
will be installed. At the end of the install, run a script that will print the package name. (Cf /yogsototh/deft/src/commit/6b27fe6c7103fc493e764ee6f31d6096438b4845/notes/digression)
And indeed, if you copy/paste this nix block in a file and run nix-shell
here is
the result:
> nix-shell
nix-shell shell.nix
these 53 paths will be fetched (84.69 MiB download, 524.77 MiB unpacked):
/nix/store/08pckaqznwh0s3822cjp5aji6y1lsm27-libcxx-11.1.0
...
/nix/store/zqcs5xahjxij0c8vfw60lnfb6d979rn2-zlib-1.2.13
copying path '/nix/store/49wn01k9yikhjlxc1ym5b6civ29zz3gv-bash-5.1-p16' from 'https://cache.nixos.org'...
...
copying path '/nix/store/4w2rv6s96fwsb4qyw8b9w394010gxriz-stdenv-darwin' from 'https://cache.nixos.org'...
Using hello-2.12.1.
[nix-shell:~/tmp/nixplayground]$
If you close the session and run it again, it will be much faster and will only show this:
❯ nix-shell
Using hello-2.12.1.
[nix-shell:~/tmp/nixplayground]$
This is because all dependencies will be cached. OK so, this is level 1 of nix-shell-fu.
Now, let's start level 2.
nix-shell-fu
level 2 lesson; scripting and configuring
This time, we want to launch a full service, as a redis docker would do.
So here is a basic shell script which is similar to the previous one but will
request redis
as a dependency instead of hello
and also as a launching script.
From there will add a little bit more features.
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz) {} }:
pkgs.mkShell {
# must contain buildInputs, nativeBuildInputs and shellHook
buildInputs = [ pkgs.redis ];
# Post Shell Hook
shellHook = ''
echo "Using ${pkgs.redis.name} on port: ${port}"
redis-server
'';
}
Again if you run nix-shell
here is the result:
❯ nix-shell
these 2 paths will be fetched (2.08 MiB download, 6.99 MiB unpacked):
/nix/store/6w4vnaxdx12ccq172i8j5l830mlp8jlg-redis-7.0.5
/nix/store/b47gmsx9qx0c9vh75wsg8bqq9qd0ad6f-openssl-3.0.7
copying path '/nix/store/b47gmsx9qx0c9vh75wsg8bqq9qd0ad6f-openssl-3.0.7' from 'https://cache.nixos.org'...
copying path '/nix/store/6w4vnaxdx12ccq172i8j5l830mlp8jlg-redis-7.0.5' from 'https://cache.nixos.org'...
Using redis-7.0.5
97814:C 10 Feb 2023 20:44:36.960 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
97814:C 10 Feb 2023 20:44:36.960 # Redis version=7.0.5, bits=64, commit=00000000, modified=0, pid=97814, just started
97814:C 10 Feb 2023 20:44:36.960 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
97814:M 10 Feb 2023 20:44:36.961 * Increased maximum number of open files to 10032 (it was originally set to 256).
97814:M 10 Feb 2023 20:44:36.961 * monotonic clock: POSIX clock_gettime
_._
_.-``__ ''-._
_.-`` `. `_. ''-._ Redis 7.0.5 (00000000/0) 64 bit
.-`` .-```. ```\/ _.,_ ''-._
( ' , .-` | `, ) Running in standalone mode
|`-._`-...-` __...-.``-._|'` _.-'| Port: 6379
| `-._ `._ / _.-' | PID: 97814
`-._ `-._ `-./ _.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' | https://redis.io
`-._ `-._`-.__.-'_.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' |
`-._ `-._`-.__.-'_.-' _.-'
`-._ `-.__.-' _.-'
`-._ _.-'
`-.__.-'
97814:M 10 Feb 2023 20:44:36.962 # WARNING: The TCP backlog setting of 511 cannot be enforced because kern.ipc.somaxconn is set to the lower value of 128.
97814:M 10 Feb 2023 20:44:36.962 # Server initialized
97814:M 10 Feb 2023 20:44:36.963 * Ready to accept connections
Woo! Redis is started and it works!
But if you have multiple projects you want to have more control. For example, we will want to run redis on a specific port. Here is how you do it:
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/21.05.tar.gz) {} }:
let iport = 16380;
port = toString iport;
in pkgs.mkShell {
# must contain buildInputs, nativeBuildInputs and shellHook
buildInputs = [ pkgs.redis ];
# Post Shell Hook
shellHook = ''
echo "Using ${pkgs.redis.name} on port ${port}"
redis-server --port ${port}
'';
}
And here is the result:
> rm dump.rdb
> nix-shell
Using redis-6.2.3 on port 16380
1785:C 10 Feb 2023 20:50:00.880 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
1785:C 10 Feb 2023 20:50:00.880 # Redis version=6.2.3, bits=64, commit=00000000, modified=0, pid=1785, just started
1785:C 10 Feb 2023 20:50:00.880 # Configuration loaded
1785:M 10 Feb 2023 20:50:00.880 * Increased maximum number of open files to 10032 (it was originally set to 256).
1785:M 10 Feb 2023 20:50:00.880 * monotonic clock: POSIX clock_gettime
_._
_.-``__ ''-._
_.-`` `. `_. ''-._ Redis 6.2.3 (00000000/0) 64 bit
.-`` .-```. ```\/ _.,_ ''-._
( ' , .-` | `, ) Running in standalone mode
|`-._`-...-` __...-.``-._|'` _.-'| Port: 16380
| `-._ `._ / _.-' | PID: 1785
`-._ `-._ `-./ _.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' | https://redis.io
`-._ `-._`-.__.-'_.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' |
`-._ `-._`-.__.-'_.-' _.-'
`-._ `-.__.-' _.-'
`-._ _.-'
`-.__.-'
1785:M 10 Feb 2023 20:50:00.881 # Server initialized
1785:M 10 Feb 2023 20:50:00.881 * Ready to accept connections
Woo! We control the port from the file.
That's nice.
But, hmmm, has you might have noticed, when you quit the session it dumps the DB
as the file dump.rdb
.
What we would like is to keep the state in a local file that would be easy to delete.
So here is how I did it, mainly, I just create a redis config file locally, and
run redis using this local config file. Also I do my best to put all files
created for running this local redis instance into a local file into my project.
The code is more complex this time, but I just added a way to create a config
file and declare a directory that will contain all the state of the DB and of
the nix configuration.
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz) {} }:
let iport = 16380;
port = toString iport;
in pkgs.mkShell (rec {
# ENV Variables the directory to put all the DATA
REDIS_DATA = "${toString ./.}/.redis";
# the config file, as we use REDIS_DATA variable we just declared in the
# same nix set, we need to use rec
redisConf = pkgs.writeText "redis.conf"
''
port ${port}
dbfilename redis.db
dir ${REDIS_DATA}
'';
buildInputs = [ pkgs.redis ];
# Post Shell Hook
shellHook = ''
echo "Using ${pkgs.redis.name} on port: ${port}"
[ ! -d $REDIS_DATA ] \
&& mkdir -p $REDIS_DATA
cat "$redisConf" > $REDIS_DATA/redis.conf
alias redisstop="echo 'Stopping Redis'; redis-cli -p ${port} shutdown; rm -rf $REDIS_DATA"
nohup redis-server $REDIS_DATA/redis.conf > /dev/null 2>&1 &
echo "When finished just run redisstop && exit"
trap redisstop EXIT
'';
})
And here is a full session using this shell.nix
:
nix-shell
Using redis-6.2.3 on port: 16380
When finished just run redisstop && exit
[nix-shell:~/tmp/nixplayground]$ redis-cli -p 16380
127.0.0.1:16380> help
redis-cli 6.2.3
To get help about Redis commands type:
"help @<group>" to get a list of commands in <group>
"help <command>" for help on <command>
"help <tab>" to get a list of possible help topics
"quit" to exit
To set redis-cli preferences:
":set hints" enable online hints
":set nohints" disable online hints
Set your preferences in ~/.redisclirc
127.0.0.1:16380>
[nix-shell:~/tmp/nixplayground]$ ls -a
. .. .redis shell.nix
[nix-shell:~/tmp/nixplayground]$ find .redis
.redis
.redis/redis.conf
[nix-shell:~/tmp/nixplayground]$ redis-cli -p 16380 shutdown
[1]+ Done nohup redis-server $REDIS_DATA/redis.conf > /dev/null 2>&1
[nix-shell:~/tmp/nixplayground]$ find .redis
.redis
.redis/redis.db
.redis/redis.conf
[nix-shell:~/tmp/nixplayground]$ redisstop
Stopping Redis
Could not connect to Redis at 127.0.0.1:16380: Connection refused
[nix-shell:~/tmp/nixplayground]$ ls -a
. .. shell.nix
[nix-shell:~/tmp/nixplayground]$
So with this version all data related to redis is saved into the local .redis
directory.
And in the nix shell we provide a command redisstop
that once invoked, shutdown
redis, then purge all redis related data (as you would like in a development environment).
Also, as compared to previous version, redis is launched in background so you
could run commands in your nix shell.
Notice I also run redisstop
command on exit of the nix-shell. So when you close
the nix-shell redis is stopped and the DB state is cleaned up.
nix-shell-fu
level 3 lesson; composability
Imagine we create another similar nix file, but this time to launch postgresql.
Roughtly, you will again build a nix set, that will contain a few env variables,
along the following entries buildInputs
, nativeBuildInputs
and shellHook
.
The issue is that in both nix files you will have the following form:
{ pkgs ? import ( ... ) {} }:
mkShell { PGDATA = ...;
buildInputs = [ dependency-1 ... dependency-n ];
nativeBuildInputs = [ dependency-1 ... dependency-n ];
shellHook = '' ... '';
}
And you cannot use that directly.
So to solve the problem, instead we will replace this format by removing mkShell
and pass the mkShell parameter instead.
We also need to be more precise about where are declared the environment
variables.
{ pkgs ? import ( ... ) {} }:
let env = { PGDATA = ...; }
in { inherit env; # equivalent to env = env;
buildInputs = [ dependency-1 ... dependency-n ];
nativeBuildInputs = [ dependency-1 ... dependency-n ];
shellHook = '' ... '';
}
With this, we can compose two nix set into a single merged one that will be
suitable for argument of mkShell.
Another minor detail, but important one. In bash, the command trap
do not
accumulate but replace the function. For our need, we want to run all stop
function on exit. So the trap
directive added in the shell hook does not compose
naturally. This is why we add a stop
value that will contain the name of the
bash function to call to stop and cleanup a service.
Finally the main structure for each of our service will look like:
{ pkgs ? import ( ... ) {} }:
let env = { PGDATA = ...; }
in { inherit env; # equivalent to env = env;
buildInputs = [ dependency-1 ... dependency-n ];
nativeBuildInputs = [ dependency-1 ... dependency-n ];
shellHook = '' ... '';
stop = "stoppostgres"
}
Mainly to merge we will just need to run:
{ pkgs ? import (...) {}}:
let
# merge all the env sets
mergedEnvs = builtins.foldl' (acc: e: acc // e) {} envs;
# merge all the confs by accumulating the dependencies
# and concatenating the shell hooks.
mergedConfs =
builtins.foldl'
(acc: {buildInputs ? [], nativeBuildInputs ? [], shellHook ? "", ...}:
{ buildInputs = acc.buildInputs ++ buildInputs;
nativeBuildInputs = acc.nativeBuildInputs ++ nativeBuildInputs;
shellHook = acc.shellHook + shellHook;
})
emptyConf
confs;
in mkShell (mergedEnvs // mergedConfs)
The full solution to deal with other minor details like importing the files, dealing with the exit of the shell is here:
{ mergeShellConfs =
# imports should contain a list of nix files
{ pkgs, imports }:
let confs = map (f: import f { inherit pkgs; }) imports;
envs = map ({env ? {}, ...}: env) confs;
# list the name of a command to stop a service (if none provided just use ':' which mean noop)
stops = map ({stop ? ":", ...}: stop) confs;
# we want to stop all services on exit
stopCmd = builtins.concatStringsSep " && " stops;
# we would like to add a shellHook to cleanup the service that will call
# all cleaning-up function declared in sub-shells
lastConf =
{ shellHook = ''
stopall() { ${stopCmd}; }
echo "You can manually stop all services by calling stopall"
trap stopall EXIT
'';
};
# merge Environment variables needed for other shell environments
mergedEnvs = builtins.foldl' (acc: e: acc // e) {} envs;
# zeroConf is the minimal empty configuration needed
zeroConf = {buildInputs = []; nativeBuildInputs = []; shellHook="";};
# merge all confs by appending buildInputs and nativeBuildInputs
# and by concatenating the shellHooks
mergedConfs =
builtins.foldl'
(acc: {buildInputs ? [], nativeBuildInputs ? [], shellHook ? "", ...}:
{ buildInputs = acc.buildInputs ++ buildInputs;
nativeBuildInputs = acc.nativeBuildInputs ++ nativeBuildInputs;
shellHook = acc.shellHook + shellHook;
})
zeroConf
(confs ++ [lastConf]);
in (mergedEnvs // mergedConfs);
}
So I put this function declaration in a file named ./nix/merge-shell.nix
.
And I have a pg.nix
as well as a redis.nix
file in the nix
directory.
On the root of the project the main shell.nix
looks like:
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz) {} }:
let
# we import the file, and rename the function mergeShellConfs as mergeShells
mergeShells = (import ./nix/merge-shell.nix).mergeShellConfs;
# we call mergeShells
mergedShellConfs =
mergeShells { inherit pkgs;
# imports = [ ./nix/pg.nix ./nix/redis.nix ];
imports = [ ./nix/pg.nix ./nix/redis.nix ];
};
in pkgs.mkShell mergedShellConfs
And, that's it.
Appendix
<<digression>> Digression
In fact, this is a bit more complex than "just that".
The reality is a bit more complex.
The nix language is "pure", meaning, if you run the nix evaluation multiple
times, it will always evaluate to the exact same value.
But here, this block represent a function.
The function takes as input a "nix set" (which you can see as an associative
array, or a hash-map or also a javascript object depending on your preference),
and this set is expected to contain a field named pkgs
. If pkgs
is not provided,
it will use the set from the stable version 22.11 of nixpkgs by downloading them
from github archive.
The second part of the function generate "something" that is returned by an
internal function of the standard library provided by nix
which is named
mkShell
.
So mainly, mkShell
is a helper function that will generate what nix calls a
derivation. Mainly, we don't really care about exactly what is a derivation.
This is an internal to nix representation that could be finally used by
different nix tools for different things. Typically, installing a package,
running a local development environment with nix-shell or nix develop, etc…
So the important detail to remember is that we can manipulate the parameter we
pass to the functions derivation
, mkDerivation
and mkShell
, but we have no
mechanism to manipulate directly derivation
. So in order to make that
composable, you need to call the derivation
internal function at the very end only.
The argument of all these functions are nix sets
The full nix files for postgres
For postgres:
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz) {} }:
let iport = 15432;
port = toString iport;
pguser = "pguser";
pgpass = "pgpass";
pgdb = "iroh";
# env should contain all variable you need to configure correctly mkShell
# so ENV_VAR, but also any other kind of variables.
env = {
postgresConf =
pkgs.writeText "postgresql.conf"
''
# Add Custom Settings
log_min_messages = warning
log_min_error_statement = error
log_min_duration_statement = 100 # ms
log_connections = on
log_disconnections = on
log_duration = on
#log_line_prefix = '[] '
log_timezone = 'UTC'
log_statement = 'all'
log_directory = 'pg_log'
log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
logging_collector = on
log_min_error_statement = error
'';
postgresInitScript =
pkgs.writeText "init.sql"
''
CREATE DATABASE ${pgdb};
CREATE USER ${pguser} WITH ENCRYPTED PASSWORD '${pgpass}';
GRANT ALL PRIVILEGES ON DATABASE ${pgdb} TO ${pguser};
'';
PGDATA = "${toString ./.}/.pg";
};
in env // {
# Warning if you add an attribute like an ENV VAR you must do it via env.
inherit env;
# must contain buildInputs, nativeBuildInputs and shellHook
buildInputs = [ pkgs.coreutils
pkgs.jdk11
pkgs.lsof
pkgs.plantuml
pkgs.leiningen
];
nativeBuildInputs = [
pkgs.zsh
pkgs.vim
pkgs.nixpkgs-fmt
pkgs.postgresql_11
# postgres-11 with postgis support
# (pkgs.postgresql_11.withPackages (p: [ p.postgis ]))
];
# Post Shell Hook
shellHook = ''
echo "Using ${pkgs.postgresql_12.name}. port: ${port} user: ${pguser} pass: ${pgpass}"
# Setup: other env variables
export PGHOST="$PGDATA"
# Setup: DB
[ ! -d $PGDATA ] \
&& pg_ctl initdb -o "-U postgres" \
&& cat "$postgresConf" >> $PGDATA/postgresql.conf
pg_ctl -o "-p ${port} -k $PGDATA" start
echo "Creating DB and User"
psql -U postgres -p ${port} -f $postgresInitScript
function pgstop {
echo "Stopping and Cleaning up Postgres";
pg_ctl stop && rm -rf $PGDATA
}
alias pg="psql -p ${port} -U postgres"
echo "Send SQL commands with pg"
trap pgstop EXIT
'';
stop = "pgstop";
}
And to just launch Posgresql, there is also this file ./nix/pgshell.nix
, that
simply contains
{ pkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz) {} }:
let pg = import ./pg.nix { inherit pkgs; };
in with pg; pkgs.mkShell ( env //
{
buildInputs = buildInputs;
nativeBuildInputs = nativeBuildInputs ;
shellHook = shellHook;
})