Merge pull request #46 from nmattia/nm-git-dep

Add support for git dependencies.
This commit is contained in:
Nicolas Mattia 2019-12-17 18:42:52 +01:00 committed by GitHub
commit d509c82968
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 144 additions and 72 deletions

View file

@ -45,10 +45,11 @@ it is converted to an attribute set equivalent to `{ root = theArg; }`.
| `version` | The version of the derivation. |
| `src` | Used by `naersk` as source input to the derivation. When `root` is not set, `src` is also used to discover the `Cargo.toml` and `Cargo.lock`. |
| `root` | Used by `naersk` to read the `Cargo.toml` and `Cargo.lock` files. May be different from `src`. When `src` is not set, `root` is (indirectly) used as `src`. |
| `cargoBuild` | The command to use for the build. Default: `''cargo build "''${cargo_release}" -j $NIX_BUILD_CORES -Z unstable-options --out-dir out''` |
| `cargoBuild` | The command to use for the build. Default: `''cargo "''${cargo_options[@]}" build "''${cargo_release[@]}" -j $NIX_BUILD_CORES -Z unstable-options --out-dir out''` |
| `doCheck` | When true, `checkPhase` is run. Default: `true` |
| `cargoTestCommands` | The commands to run in the `checkPhase`. Default: `[ ''cargo test "''${cargo_release}" -j $NIX_BUILD_CORES'' ]` |
| `cargoTestCommands` | The commands to run in the `checkPhase`. Default: `[ ''cargo "''${cargo_options[@]}" test "''${cargo_release[@]}" -j $NIX_BUILD_CORES'' ]` |
| `buildInputs` | Extra `buildInputs` to all derivations. Default: `[]` |
| `cargoOptions` | Options passed to cargo before the command (cargo OPTIONS <cmd>) Default: `[]` |
| `doDoc` | When true, `cargo doc` is run and a new output `doc` is generated. Default: `true` |
| `release` | When true, all cargo builds are run with `--release`. Default: `true` |
| `override` | An override for all derivations involved in the build. Default: `(x: x)` |
@ -73,8 +74,7 @@ project's `Cargo.lock` which makes any code generation unnecessary.
For the same reason, `naersk` does not need anything like `rustPlatform`'s
`cargoSha256`. All crates are downloaded using the `sha256` checksums provided
in the project's `Cargo.lock`. Because `cargo` doesn't register checksums for
`git` dependencies, **`naersk` does not support `git` dependencies**.
in the project's `Cargo.lock`.
Finally `naersk` supports incremental builds by first performing a
dependency-only build, and _then_ a build that depends on the top-level crate's

View file

@ -52,8 +52,7 @@ project's `Cargo.lock` which makes any code generation unnecessary.
For the same reason, `naersk` does not need anything like `rustPlatform`'s
`cargoSha256`. All crates are downloaded using the `sha256` checksums provided
in the project's `Cargo.lock`. Because `cargo` doesn't register checksums for
`git` dependencies, **`naersk` does not support `git` dependencies**.
in the project's `Cargo.lock`.
Finally `naersk` supports incremental builds by first performing a
dependency-only build, and _then_ a build that depends on the top-level crate's

View file

@ -29,6 +29,7 @@
# Which drops the run-time dependency on the crates-io source thereby
# significantly reducing the Nix closure size.
, removeReferencesToSrcFromDocs
, gitDependencies
, pname
, version
, rustc
@ -37,6 +38,7 @@
, buildInputs
, builtDependencies
, release
, cargoOptions
, stdenv
, lib
, rsync
@ -56,6 +58,55 @@ let
builtins // import ./builtins
{ inherit lib writeText remarshal runCommand; };
# All the git dependencies, as a list
gitDependenciesList =
lib.concatLists (lib.mapAttrsToList (_: ds: ds) gitDependencies);
# This unpacks all git dependencies:
# $out/rand
# $out/rand/Cargo.toml
# $out/rand_core
# ...
# It does so by discovering all the `Cargo.toml`s and creating a directory in
# $out for each one.
# NOTE:
# Only non-virtual manifests are taken into account. That is, only cargo
# tomls that have a [package] sections with a `name = ...`. The
# implementation is a bit tricky and basically akin to parsing TOML with
# bash. The reason is that there is no lightweight jq-equivalent available
# in nixpkgs (rq fails to build).
# We discover the name (in any) in three steps:
# * grab anything that comes after `[package]`
# * grab the first line that contains `name = ...`
# * grab whatever is surrounded with `"`s.
# The last step is very, very slow.
unpackedGitDependencies = runCommand "git-deps"
{ nativeBuildInputs = [ jq ]; }
''
mkdir -p $out
while read -r dep; do
checkout=$(echo "$dep" | jq -cMr '.checkout')
url=$(echo "$dep" | jq -cMr '.url')
tomls=$(find $checkout -name Cargo.toml)
while read -r toml; do
name=$(cat $toml \
| sed -n -e '/\[package\]/,$p' \
| grep -m 1 "^name\W" \
| grep -oP '(?<=").+(?=")' \
|| true)
if [ -n "$name" ]; then
echo "$url Found crate '$name'"
cp -r $(dirname $toml) $out/$name
chmod +w $out/$name
echo '{"package":null,"files":{}}' > $out/$name/.cargo-checksum.json
fi
done <<< "$tomls"
done < <(cat ${
builtins.toFile "git-deps-json" (builtins.toJSON gitDependenciesList)
} | jq -cMr '.[]')
'';
drv = stdenv.mkDerivation {
name = "${pname}-${version}";
inherit
@ -65,6 +116,8 @@ let
preBuild
;
# The cargo config with source replacement. Replaces both crates.io crates
# and git dependencies.
cargoconfig = builtinz.toTOML {
source = {
crates-io = { replace-with = "nix-sources"; };
@ -72,10 +125,19 @@ let
directory = symlinkJoin {
name = "crates-io";
paths = map (v: unpackCrate v.name v.version v.sha256)
crateDependencies;
crateDependencies ++ [ unpackedGitDependencies ] ;
};
};
};
} // lib.listToAttrs ( map
(e:
{ name = e.url; value =
{ git = e.url;
rev = e.rev;
replace-with = "nix-sources";
};
})
gitDependenciesList
);
};
outputs = [ "out" ] ++ lib.optional (doDoc && copyDocsToSeparateOutput) "doc";
@ -104,6 +166,7 @@ let
configurePhase = ''
cargo_release=( ${lib.optionalString release "--release" } )
cargo_options=( ${lib.escapeShellArgs cargoOptions} )
runHook preConfigure

View file

@ -14,14 +14,16 @@ let
root = attrs0.root or null;
# The command to use for the build.
cargoBuild = attrs0.cargoBuild or
''cargo build "''${cargo_release}" -j $NIX_BUILD_CORES -Z unstable-options --out-dir out'';
''cargo "''${cargo_options[@]}" build "''${cargo_release[@]}" -j $NIX_BUILD_CORES -Z unstable-options --out-dir out'';
# When true, `checkPhase` is run.
doCheck = attrs0.doCheck or true;
# The commands to run in the `checkPhase`.
cargoTestCommands = attrs0.cargoTestCommands or
[ ''cargo test "''${cargo_release}" -j $NIX_BUILD_CORES'' ];
[ ''cargo "''${cargo_options[@]}" test "''${cargo_release[@]}" -j $NIX_BUILD_CORES'' ];
# Extra `buildInputs` to all derivations.
buildInputs = attrs0.buildInputs or [];
# Options passed to cargo before the command (cargo OPTIONS <cmd>)
cargoOptions = attrs0.cargoOptions or [];
# When true, `cargo doc` is run and a new output `doc` is generated.
doDoc = attrs0.doDoc or true;
# When true, all cargo builds are run with `--release`.
@ -109,6 +111,7 @@ let
buildConfig = {
compressTarget = attrs.compressTarget;
doCheck = attrs.doCheck;
cargoOptions = attrs.cargoOptions;
buildInputs = attrs.buildInputs;
removeReferencesToSrcFromDocs = attrs.removeReferencesToSrcFromDocs;
doDoc = attrs.doDoc;
@ -134,7 +137,7 @@ let
buildPlanConfig = rec {
inherit (sr) src root;
# Whether we skip pre-building the deps
isSingleStep = attrs.singleStep or false;
isSingleStep = attrs.singleStep;
# The members we want to build
# (list of directory names)

View file

@ -45,6 +45,8 @@ let
buildPackage = arg:
let
config = mkConfig arg;
gitDependencies =
libb.findGitDependencies { inherit (config) cargotomls; };
in
import ./build.nix
(
@ -58,12 +60,14 @@ let
if [ -f src/main.rs ] ; then touch src/main.rs; fi
'';
inherit (config) src cargoTestCommands copyTarget copyBins copyDocsToSeparateOutput;
inherit gitDependencies;
} // config.buildConfig // {
builtDependencies = lib.optional (! config.isSingleStep)
(
import ./build.nix
(
{
inherit gitDependencies;
src = libb.dummySrc {
cargoconfig =
if builtinz.pathExists (toString config.root + "/.cargo/config")

33
lib.nix
View file

@ -56,6 +56,39 @@ rec
mkMetadataKey = name: version:
"checksum ${name} ${version} (registry+https://github.com/rust-lang/crates.io-index)";
# a record:
# { "." = # '.' is the directory of the cargotoml
# [
# {
# name = "rand";
# url = "https://github.com/...";
# checkout = "/nix/store/checkout"
# }
# ]
# }
findGitDependencies =
{ cargotomls
}:
let
tomlDependencies = cargotoml:
lib.filter (x: ! isNull x) (
lib.mapAttrsToList
(k: v:
if ! (lib.isAttrs v && builtins.hasAttr "git" v)
then null
else
{ name = k;
url = v.git;
rev = v.rev;
checkout = builtins.fetchGit {
url = v.git;
rev = v.rev;
};
}
) cargotoml.dependencies or {});
in
lib.mapAttrs (_: tomlDependencies) cargotomls;
# A very minimal 'src' which makes cargo happy nonetheless
dummySrc =
{ cargoconfig # string

View file

@ -16,6 +16,8 @@ rec
'';
docparse = naersk.buildPackage {
doDoc = false;
doCheck = false;
root = ./docparse;
src = builtins.filterSource (
p: t:
@ -113,6 +115,11 @@ rec
{ buildInputs = [ dummyfication ]; }
"my-bin > $out";
git-dep = naersk.buildPackage {
root = ./test/git-dep;
cargoOptions = [ "--locked" ];
};
workspace = naersk.buildPackage {
src = ./test/workspace;
doDoc = false;

View file

@ -1,39 +1,26 @@
[[package]]
name = "bitflags"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "c2-chacha"
version = "0.2.2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "getrandom"
version = "0.1.3"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -43,19 +30,14 @@ dependencies = [
"rand 0.7.0-pre.1 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)",
]
[[package]]
name = "lazy_static"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.58"
version = "0.2.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "ppv-lite86"
version = "0.2.5"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -63,8 +45,8 @@ name = "rand"
version = "0.7.0-pre.1"
source = "git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60#703452450770d4b2bb0b117dfc83aea5ba61ec60"
dependencies = [
"getrandom 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
"getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_chacha 0.2.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)",
"rand_core 0.5.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)",
"rand_hc 0.2.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)",
@ -75,7 +57,7 @@ name = "rand_chacha"
version = "0.2.0"
source = "git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60#703452450770d4b2bb0b117dfc83aea5ba61ec60"
dependencies = [
"c2-chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.5.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)",
]
@ -84,7 +66,7 @@ name = "rand_core"
version = "0.5.0"
source = "git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60#703452450770d4b2bb0b117dfc83aea5ba61ec60"
dependencies = [
"getrandom 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -96,37 +78,18 @@ dependencies = [
]
[[package]]
name = "winapi"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
name = "wasi"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
"checksum c2-chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7d64d04786e0f528460fc884753cf8dddcc466be308f6026f8e355c41a0e4101"
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
"checksum getrandom 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8d1dffef07351aafe6ef177e4dd2b8dcf503e6bc765dea3b0de9ed149a3db1ec"
"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14"
"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319"
"checksum ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e3cbf9f658cdb5000fcf6f362b8ea2ba154b9f146a61c7a20d647034c6b6561b"
"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
"checksum rand 0.7.0-pre.1 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)" = "<none>"
"checksum rand_chacha 0.2.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)" = "<none>"
"checksum rand_core 0.5.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)" = "<none>"
"checksum rand_hc 0.2.0 (git+https://github.com/rust-random/rand?rev=703452450770d4b2bb0b117dfc83aea5ba61ec60)" = "<none>"
"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"