3
0
Fork 0
forked from mirrors/nixpkgs

Merge pull request #112477 from happysalada/fix_build_mix

buildMix: fix: initial try
This commit is contained in:
Alexander Bantyev 2021-04-09 12:09:22 +03:00 committed by GitHub
commit 544664d484
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 400 additions and 288 deletions

View file

@ -2,15 +2,15 @@
## Introduction {#beam-introduction}
In this document and related Nix expressions, we use the term, *BEAM*, to describe the environment. BEAM is the name of the Erlang Virtual Machine and, as far as we're concerned, from a packaging perspective, all languages that run on the BEAM are interchangeable. That which varies, like the build system, is transparent to users of any given BEAM package, so we make no distinction.
In this document and related Nix expressions, we use the term, _BEAM_, to describe the environment. BEAM is the name of the Erlang Virtual Machine and, as far as we're concerned, from a packaging perspective, all languages that run on the BEAM are interchangeable. That which varies, like the build system, is transparent to users of any given BEAM package, so we make no distinction.
## Structure {#beam-structure}
All BEAM-related expressions are available via the top-level `beam` attribute, which includes:
- `interpreters`: a set of compilers running on the BEAM, including multiple Erlang/OTP versions (`beam.interpreters.erlangR19`, etc), Elixir (`beam.interpreters.elixir`) and LFE (`beam.interpreters.lfe`).
- `interpreters`: a set of compilers running on the BEAM, including multiple Erlang/OTP versions (`beam.interpreters.erlangR19`, etc), Elixir (`beam.interpreters.elixir`) and LFE (Lisp Flavoured Erlang) (`beam.interpreters.lfe`).
- `packages`: a set of package builders (Mix and rebar3), each compiled with a specific Erlang/OTP version, e.g. `beam.packages.erlangR19`.
- `packages`: a set of package builders (Mix and rebar3), each compiled with a specific Erlang/OTP version, e.g. `beam.packages.erlangR19`.
The default Erlang compiler, defined by `beam.interpreters.erlang`, is aliased as `erlang`. The default BEAM package set is defined by `beam.packages.erlang` and aliased at the top level as `beamPackages`.
@ -26,7 +26,9 @@ We provide a version of Rebar3, under `rebar3`. We also provide a helper to fetc
### Mix & Erlang.mk {#build-tools-other}
Both Mix and Erlang.mk work exactly as expected. There is a bootstrap process that needs to be run for both, however, which is supported by the `buildMix` and `buildErlangMk` derivations, respectively.
Erlang.mk works exactly as expected. There is a bootstrap process that needs to be run, which is supported by the `buildErlangMk` derivation.
For Elixir applications use `mixRelease` to make a release. See examples for more details.
## How to Install BEAM Packages {#how-to-install-beam-packages}
@ -52,15 +54,150 @@ Erlang.mk functions similarly to Rebar3, except we use `buildErlangMk` instead o
#### Mix Packages {#mix-packages}
Mix functions similarly to Rebar3, except we use `buildMix` instead of `buildRebar3`.
`mixRelease` is used to make a release in the mix sense. Dependencies will need to be fetched with `fetchMixDeps` and passed to it.
Alternatively, we can use `buildHex` as a shortcut:
#### mixRelease - Elixir Phoenix example
Here is how your `default.nix` file would look.
```nix
with import <nixpkgs> { };
let
packages = beam.packagesWith beam.interpreters.erlang;
src = builtins.fetchgit {
url = "ssh://git@github.com/your_id/your_repo";
rev = "replace_with_your_commit";
};
pname = "your_project";
version = "0.0.1";
mixEnv = "prod";
mixDeps = packages.fetchMixDeps {
pname = "mix-deps-${pname}";
inherit src mixEnv version;
# nix will complain and tell you the right value to replace this with
sha256 = lib.fakeSha256;
# if you have build time environment variables add them here
MY_ENV_VAR="my_value";
};
nodeDependencies = (pkgs.callPackage ./assets/default.nix { }).shell.nodeDependencies;
frontEndFiles = stdenvNoCC.mkDerivation {
pname = "frontend-${pname}";
nativeBuildInputs = [ nodejs ];
inherit version src;
buildPhase = ''
cp -r ./assets $TEMPDIR
mkdir -p $TEMPDIR/assets/node_modules/.cache
cp -r ${nodeDependencies}/lib/node_modules $TEMPDIR/assets
export PATH="${nodeDependencies}/bin:$PATH"
cd $TEMPDIR/assets
webpack --config ./webpack.config.js
cd ..
'';
installPhase = ''
cp -r ./priv/static $out/
'';
outputHashAlgo = "sha256";
outputHashMode = "recursive";
# nix will complain and tell you the right value to replace this with
outputHash = lib.fakeSha256;
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
};
in packages.mixRelease {
inherit src pname version mixEnv mixDeps;
# if you have build time environment variables add them here
MY_ENV_VAR="my_value";
preInstall = ''
mkdir -p ./priv/static
cp -r ${frontEndFiles} ./priv/static
'';
}
```
Setup will require the following steps:
- Move your secrets to runtime environment variables. For more information refer to the [runtime.exs docs](https://hexdocs.pm/mix/Mix.Tasks.Release.html#module-runtime-configuration). On a fresh Phoenix build that would mean that both `DATABASE_URL` and `SECRET_KEY` need to be moved to `runtime.exs`.
- `cd assets` and `nix-shell -p node2nix --run node2nix --development` will generate a Nix expression containing your frontend dependencies
- commit and push those changes
- you can now `nix-build .`
- To run the release, set the `RELEASE_TMP` environment variable to a directory that your program has write access to. It will be used to store the BEAM settings.
#### Example of creating a service for an Elixir - Phoenix project
In order to create a service with your release, you could add a `service.nix`
in your project with the following
```nix
{config, pkgs, lib, ...}:
let
release = pkgs.callPackage ./default.nix;
release_name = "app";
working_directory = "/home/app";
in
{
systemd.services.${release_name} = {
wantedBy = [ "multi-user.target" ];
after = [ "network.target" "postgresql.service" ];
requires = [ "network-online.target" "postgresql.service" ];
description = "my app";
environment = {
# RELEASE_TMP is used to write the state of the
# VM configuration when the system is running
# it needs to be a writable directory
RELEASE_TMP = working_directory;
# can be generated in an elixir console with
# Base.encode32(:crypto.strong_rand_bytes(32))
RELEASE_COOKIE = "my_cookie";
MY_VAR = "my_var";
};
serviceConfig = {
Type = "exec";
DynamicUser = true;
WorkingDirectory = working_directory;
# Implied by DynamicUser, but just to emphasize due to RELEASE_TMP
PrivateTmp = true;
ExecStart = ''
${release}/bin/${release_name} start
'';
ExecStop = ''
${release}/bin/${release_name} stop
'';
ExecReload = ''
${release}/bin/${release_name} restart
'';
Restart = "on-failure";
RestartSec = 5;
StartLimitBurst = 3;
StartLimitInterval = 10;
};
# disksup requires bash
path = [ pkgs.bash ];
};
environment.systemPackages = [ release ];
}
```
## How to Develop {#how-to-develop}
### Creating a Shell {#creating-a-shell}
Usually, we need to create a `shell.nix` file and do our development inside of the environment specified therein. Just install your version of erlang and other interpreter, and then user your normal build tools. As an example with elixir:
Usually, we need to create a `shell.nix` file and do our development inside of the environment specified therein. Just install your version of Erlang and any other interpreters, and then use your normal build tools. As an example with Elixir:
```nix
{ pkgs ? import "<nixpkgs"> {} }:
@ -79,6 +216,68 @@ mkShell {
}
```
#### Building in a Shell (for Mix Projects) {#building-in-a-shell}
#### Elixir - Phoenix project
Using a `shell.nix` as described (see <xref linkend="creating-a-shell"/>) should just work.
Here is an example `shell.nix`.
```nix
with import <nixpkgs> { };
let
# define packages to install
basePackages = [
git
# replace with beam.packages.erlang.elixir_1_11 if you need
beam.packages.erlang.elixir
nodejs-15_x
postgresql_13
# only used for frontend dependencies
# you are free to use yarn2nix as well
nodePackages.node2nix
# formatting js file
nodePackages.prettier
];
inputs = basePackages ++ lib.optionals stdenv.isLinux [ inotify-tools ]
++ lib.optionals stdenv.isDarwin
(with darwin.apple_sdk.frameworks; [ CoreFoundation CoreServices ]);
# define shell startup command
hooks = ''
# this allows mix to work on the local directory
mkdir -p .nix-mix .nix-hex
export MIX_HOME=$PWD/.nix-mix
export HEX_HOME=$PWD/.nix-mix
export PATH=$MIX_HOME/bin:$HEX_HOME/bin:$PATH
# TODO: not sure how to make hex available without installing it afterwards.
mix local.hex --if-missing
export LANG=en_US.UTF-8
export ERL_AFLAGS="-kernel shell_history enabled"
# postges related
# keep all your db data in a folder inside the project
export PGDATA="$PWD/db"
# phoenix related env vars
export POOL_SIZE=15
export DB_URL="postgresql://postgres:postgres@localhost:5432/db"
export PORT=4000
export MIX_ENV=dev
# add your project env vars here, word readable in the nix store.
export ENV_VAR="your_env_var"
'';
in mkShell {
buildInputs = inputs;
shellHook = hooks;
}
```
Initializing the project will require the following steps:
- create the db directory `initdb ./db` (inside your mix project folder)
- create the postgres user `createuser postgres -ds`
- create the db `createdb db`
- start the postgres instance `pg_ctl -l "$PGDATA/server.log" start`
- add the `/db` folder to your `.gitignore`
- you can start your phoenix server and get a shell with `iex -S mix phx.server`

View file

@ -1,100 +0,0 @@
{ stdenv, writeText, elixir, erlang, hex, lib }:
{ name
, version
, src
, setupHook ? null
, buildInputs ? []
, beamDeps ? []
, postPatch ? ""
, compilePorts ? false
, installPhase ? null
, buildPhase ? null
, configurePhase ? null
, meta ? {}
, enableDebugInfo ? false
, ... }@attrs:
with lib;
let
debugInfoFlag = lib.optionalString (enableDebugInfo || elixir.debugInfo) "--debug-info";
shell = drv: stdenv.mkDerivation {
name = "interactive-shell-${drv.name}";
buildInputs = [ drv ];
};
bootstrapper = ./mix-bootstrap;
pkg = self: stdenv.mkDerivation ( attrs // {
name = "${name}-${version}";
inherit version;
dontStrip = true;
inherit src;
setupHook = if setupHook == null
then writeText "setupHook.sh" ''
addToSearchPath ERL_LIBS "$1/lib/erlang/lib"
''
else setupHook;
inherit buildInputs;
propagatedBuildInputs = [ hex elixir ] ++ beamDeps;
configurePhase = if configurePhase == null
then ''
runHook preConfigure
${erlang}/bin/escript ${bootstrapper}
runHook postConfigure
''
else configurePhase ;
buildPhase = if buildPhase == null
then ''
runHook preBuild
export HEX_OFFLINE=1
export HEX_HOME=`pwd`
export MIX_ENV=prod
export MIX_NO_DEPS=1
mix compile ${debugInfoFlag} --no-deps-check
runHook postBuild
''
else buildPhase;
installPhase = if installPhase == null
then ''
runHook preInstall
MIXENV=prod
if [ -d "_build/shared" ]; then
MIXENV=shared
fi
mkdir -p "$out/lib/erlang/lib/${name}-${version}"
for reldir in src ebin priv include; do
fd="_build/$MIXENV/lib/${name}/$reldir"
[ -d "$fd" ] || continue
cp -Hrt "$out/lib/erlang/lib/${name}-${version}" "$fd"
success=1
done
runHook postInstall
''
else installPhase;
passthru = {
packageName = name;
env = shell self;
inherit beamDeps;
};
});
in fix pkg

View file

@ -3,7 +3,7 @@
let
inherit (lib) makeExtensible;
lib' = pkgs.callPackage ./lib.nix {};
lib' = pkgs.callPackage ./lib.nix { };
# FIXME: add support for overrideScope
callPackageWithScope = scope: drv: args: lib.callPackageWith scope drv args;
@ -22,19 +22,18 @@ let
rebar3 = callPackage ../tools/build-managers/rebar3 { };
# rebar3 port compiler plugin is required by buildRebar3
pc_1_6_0 = callPackage ./pc {};
pc = pc_1_6_0;
pc = callPackage ./pc { };
fetchHex = callPackage ./fetch-hex.nix { };
fetchRebar3Deps = callPackage ./fetch-rebar-deps.nix { };
rebar3Relx = callPackage ./rebar3-release.nix { };
buildRebar3 = callPackage ./build-rebar3.nix {};
buildHex = callPackage ./build-hex.nix {};
buildErlangMk = callPackage ./build-erlang-mk.nix {};
buildRebar3 = callPackage ./build-rebar3.nix { };
buildHex = callPackage ./build-hex.nix { };
buildErlangMk = callPackage ./build-erlang-mk.nix { };
fetchMixDeps = callPackage ./fetch-mix-deps.nix { };
buildMix = callPackage ./build-mix.nix {};
mixRelease = callPackage ./mix-release.nix { };
# BEAM-based languages.
elixir = elixir_1_11;
@ -73,11 +72,12 @@ let
# Non hex packages. Examples how to build Rebar/Mix packages with and
# without helper functions buildRebar3 and buildMix.
hex = callPackage ./hex {};
webdriver = callPackage ./webdriver {};
relxExe = callPackage ../tools/erlang/relx-exe {};
hex = callPackage ./hex { };
webdriver = callPackage ./webdriver { };
relxExe = callPackage ../tools/erlang/relx-exe { };
# An example of Erlang/C++ package.
cuter = callPackage ../tools/erlang/cuter {};
cuter = callPackage ../tools/erlang/cuter { };
};
in makeExtensible packages
in
makeExtensible packages

View file

@ -1,34 +1,49 @@
{ stdenvNoCC, lib, elixir, hex, rebar, rebar3, cacert, git }:
{ name, version, sha256, src, mixEnv ? "prod", debug ? false, meta ? { } }:
stdenvNoCC.mkDerivation ({
name = "mix-deps-${name}-${version}";
{ pname
, version
, sha256
, src
, mixEnv ? "prod"
, debug ? false
, meta ? { }
, ...
}@attrs:
stdenvNoCC.mkDerivation (attrs // {
nativeBuildInputs = [ elixir hex cacert git ];
inherit src;
MIX_ENV = mixEnv;
MIX_DEBUG = if debug then 1 else 0;
DEBUG = if debug then 1 else 0; # for rebar3
# the api with `mix local.rebar rebar path` makes a copy of the binary
MIX_REBAR = "${rebar}/bin/rebar";
MIX_REBAR3 = "${rebar3}/bin/rebar3";
# there is a persistent download failure with absinthe 1.6.3
# those defaults reduce the failure rate
HEX_HTTP_CONCURRENCY = 1;
HEX_HTTP_TIMEOUT = 120;
configurePhase = ''
configurePhase = attrs.configurePhase or ''
runHook preConfigure
export HEX_HOME="$TEMPDIR/.hex";
export MIX_HOME="$TEMPDIR/.mix";
export MIX_DEPS_PATH="$out";
export MIX_DEPS_PATH="$TEMPDIR/deps";
# Rebar
mix local.rebar rebar "${rebar}/bin/rebar"
mix local.rebar rebar3 "${rebar3}/bin/rebar3"
export REBAR_GLOBAL_CONFIG_DIR="$TMPDIR/rebar3"
export REBAR_CACHE_DIR="$TMPDIR/rebar3.cache"
runHook postConfigure
'';
dontBuild = true;
installPhase = ''
installPhase = attrs.installPhase or ''
runHook preInstall
mix deps.get --only ${mixEnv}
find "$TEMPDIR/deps" -path '*/.git/*' -a ! -name HEAD -exec rm -rf {} +
cp -r --no-preserve=mode,ownership,timestamps $TEMPDIR/deps $out
runHook postInstall
'';
outputHashAlgo = "sha256";

View file

@ -1,108 +0,0 @@
#!/usr/bin/env escript
%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%%! -smp enable
%%% ---------------------------------------------------------------------------
%%% @doc
%%% The purpose of this command is to prepare a mix project so that mix
%%% understands that the dependencies are all already installed. If you want a
%%% hygienic build on nix then you must run this command before running mix. I
%%% suggest that you add a `Makefile` to your project and have the bootstrap
%%% command be a dependency of the build commands. See the nix documentation for
%%% more information.
%%%
%%% This command designed to have as few dependencies as possible so that it can
%%% be a dependency of root level packages like mix. To that end it does many
%%% things in a fairly simplistic way. That is by design.
%%%
%%% ### Assumptions
%%%
%%% This command makes the following assumptions:
%%%
%%% * It is run in a nix-shell or nix-build environment
%%% * that all dependencies have been added to the ERL_LIBS
%%% Environment Variable
-record(data, {version
, erl_libs
, root
, name}).
-define(LOCAL_HEX_REGISTRY, "registry.ets").
main(Args) ->
{ok, RequiredData} = gather_required_data_from_the_environment(Args),
ok = bootstrap_libs(RequiredData).
%% @doc
%% This takes an app name in the standard OTP <name>-<version> format
%% and returns just the app name. Why? Because rebar doesn't
%% respect OTP conventions in some cases.
-spec fixup_app_name(file:name()) -> string().
fixup_app_name(Path) ->
BaseName = filename:basename(Path),
case string:split(BaseName, "-") of
[Name, _Version] -> Name;
Name -> Name
end.
-spec gather_required_data_from_the_environment([string()]) -> {ok, #data{}}.
gather_required_data_from_the_environment(_) ->
{ok, #data{ version = guard_env("version")
, erl_libs = os:getenv("ERL_LIBS", [])
, root = code:root_dir()
, name = guard_env("name")}}.
-spec guard_env(string()) -> string().
guard_env(Name) ->
case os:getenv(Name) of
false ->
stderr("Expected Environment variable ~s! Are you sure you are "
"running in a Nix environment? Either a nix-build, "
"nix-shell, etc?~n", [Name]),
erlang:halt(1);
Variable ->
Variable
end.
-spec bootstrap_libs(#data{}) -> ok.
bootstrap_libs(#data{erl_libs = ErlLibs}) ->
io:format("Bootstrapping dependent libraries~n"),
Target = "_build/prod/lib/",
Paths = string:tokens(ErlLibs, ":"),
CopiableFiles =
lists:foldl(fun(Path, Acc) ->
gather_directory_contents(Path) ++ Acc
end, [], Paths),
lists:foreach(fun (Path) ->
ok = link_app(Path, Target)
end, CopiableFiles).
-spec gather_directory_contents(string()) -> [{string(), string()}].
gather_directory_contents(Path) ->
{ok, Names} = file:list_dir(Path),
lists:map(fun(AppName) ->
{filename:join(Path, AppName), fixup_app_name(AppName)}
end, Names).
%% @doc
%% Makes a symlink from the directory pointed at by Path to a
%% directory of the same name in Target. So if we had a Path of
%% {`foo/bar/baz/bash`, `baz`} and a Target of `faz/foo/foos`, the symlink
%% would be `faz/foo/foos/baz`.
-spec link_app({string(), string()}, string()) -> ok.
link_app({Path, TargetFile}, TargetDir) ->
Target = filename:join(TargetDir, TargetFile),
ok = make_symlink(Path, Target).
-spec make_symlink(string(), string()) -> ok.
make_symlink(Path, TargetFile) ->
file:delete(TargetFile),
ok = filelib:ensure_dir(TargetFile),
io:format("Making symlink from ~s to ~s~n", [Path, TargetFile]),
ok = file:make_symlink(Path, TargetFile).
%% @doc
%% Write the result of the format string out to stderr.
-spec stderr(string(), [term()]) -> ok.
stderr(FormatStr, Args) ->
io:put_chars(standard_error, io_lib:format(FormatStr, Args)).

View file

@ -0,0 +1,106 @@
{ stdenv, lib, elixir, erlang, findutils, hex, rebar, rebar3, fetchMixDeps, makeWrapper, git }:
{ pname
, version
, src
, nativeBuildInputs ? [ ]
, meta ? { }
, enableDebugInfo ? false
, mixEnv ? "prod"
, compileFlags ? [ ]
, mixDeps ? null
, ...
}@attrs:
let
overridable = builtins.removeAttrs attrs [ "compileFlags" ];
in
stdenv.mkDerivation (overridable // {
nativeBuildInputs = nativeBuildInputs ++ [ erlang hex elixir makeWrapper git ];
MIX_ENV = mixEnv;
MIX_DEBUG = if enableDebugInfo then 1 else 0;
HEX_OFFLINE = 1;
DEBUG = if enableDebugInfo then 1 else 0; # for Rebar3 compilation
# the api with `mix local.rebar rebar path` makes a copy of the binary
MIX_REBAR = "${rebar}/bin/rebar";
MIX_REBAR3 = "${rebar3}/bin/rebar3";
postUnpack = ''
export HEX_HOME="$TEMPDIR/hex"
export MIX_HOME="$TEMPDIR/mix"
# compilation of the dependencies will require
# that the dependency path is writable
# thus a copy to the TEMPDIR is inevitable here
export MIX_DEPS_PATH="$TEMPDIR/deps"
# Rebar
export REBAR_GLOBAL_CONFIG_DIR="$TEMPDIR/rebar3"
export REBAR_CACHE_DIR="$TEMPDIR/rebar3.cache"
${lib.optionalString (mixDeps != null) ''
cp --no-preserve=mode -R "${mixDeps}" "$MIX_DEPS_PATH"
''
}
'' + (attrs.postUnpack or "");
configurePhase = attrs.configurePhase or ''
runHook preConfigure
# this is needed for projects that have a specific compile step
# the dependency needs to be compiled in order for the task
# to be available
# Phoenix projects for example will need compile.phoenix
mix deps.compile --no-deps-check --skip-umbrella-children
runHook postConfigure
'';
buildPhase = attrs.buildPhase or ''
runHook preBuild
mix compile --no-deps-check ${lib.concatStringsSep " " compileFlags}
runHook postBuild
'';
installPhase = attrs.installPhase or ''
runHook preInstall
mix release --no-deps-check --path "$out"
runHook postInstall
'';
fixupPhase = ''
runHook preFixup
if [ -e "$out/bin/${pname}.bat" ]; then # absent in special cases, i.e. elixir-ls
rm "$out/bin/${pname}.bat" # windows file
fi
# contains secrets and should not be in the nix store
# TODO document how to handle RELEASE_COOKIE
# secrets should not be in the nix store.
# This is only used for connecting multiple nodes
if [ -e $out/releases/COOKIE ]; then # absent in special cases, i.e. elixir-ls
rm $out/releases/COOKIE
fi
# TODO remove the uneeded reference too erlang
# one possible way would be
# for f in $(${findutils}/bin/find $out -name start); do
# substituteInPlace $f \
# --replace 'ROOTDIR=${erlang}/lib/erlang' 'ROOTDIR=""'
# done
# What is left to do is to check that erlang is not required on
# the host
patchShebangs $out
runHook postFixup
'';
# TODO figure out how to do a Fixed Output Derivation and add the output hash
# This doesn't play well at the moment with Phoenix projects
# for example that have frontend dependencies
# disallowedReferences = [ erlang ];
})