Fix features rebuild

build.nix

@@ -2,11 +2,13 @@ src:
 { #| What command to run during the build phase
   cargoBuild
 , #| What command to run during the test phase
-  cargoTest ? "cargo test --$CARGO_BUILD_PROFILE"
+  cargoTest
   #| Whether or not to forward build artifacts to $out
 , copyBuildArtifacts ? false
 , doCheck ? true
 , doDoc ? true
+  #| Whether or not the rustdoc can fail the build
+, doDocFail ? false
 , copyDocsToSeparateOutput ? true
   #| Whether to remove references to source code from the generated cargo docs
   #  to reduce Nix closure size. By default cargo doc includes snippets like the
@@ -32,8 +34,7 @@ src:
 , buildInputs ? []
 , nativeBuildInputs ? []
 , builtDependencies ? []
-, cargolock ? null
+, replaceToml ? true
-, cargotoml ? null
 , release ? true
 , stdenv
 , lib
@@ -46,6 +47,7 @@ src:
 , runCommand
 , remarshal
 , crateDependencies
+# TODO: rename to "members"
 , cratePaths
 }:
 
@@ -67,18 +69,18 @@ with rec
           name
           version;
 
-      cargoconfig = builtinz.toTOML
+        cargoconfig = builtinz.toTOML
-        { source =
+          { source =
-            { crates-io = { replace-with = "nix-sources"; } ;
+              { crates-io = { replace-with = "nix-sources"; } ;
-              nix-sources =
+                nix-sources =
-                { directory = symlinkJoin
+                  { directory = symlinkJoin
-                    { name = "crates-io";
+                      { name = "crates-io";
-                      paths = map (v: unpackCrate v.name v.version v.sha256)
+                        paths = map (v: unpackCrate v.name v.version v.sha256)
-                        crateDependencies;
+                          crateDependencies;
-                    };
+                      };
-                };
+                  };
-            };
+              };
-        };
+          };
 
         outputs = [ "out" ] ++ lib.optional (doDoc && copyDocsToSeparateOutput) "doc";
         preInstallPhases = lib.optional doDoc [ "docPhase" ];
@@ -114,26 +116,6 @@ with rec
           ''
             runHook preConfigure
 
-            if [ -n "$cargolock" ]
-            then
-              echo "Setting Cargo.lock"
-              if [ -f "Cargo.lock" ]
-              then
-                echo "WARNING: replacing existing Cargo.lock"
-              fi
-              install -m 644 "$cargolock" Cargo.lock
-            fi
-
-            if [ -n "$cargotoml" ]
-            then
-              echo "Setting Cargo.toml"
-              if [ -f "Cargo.toml" ]
-              then
-                echo "WARNING: replacing existing Cargo.toml"
-              fi
-              install -m 644 "$cargotoml" Cargo.toml
-            fi
-
             mkdir -p target
 
             cat ${builtinz.writeJSON "dependencies-json" builtDependencies} |\
@@ -165,8 +147,7 @@ with rec
           ''
             runHook preBuild
 
-            echo "Running build command:"
+            echo "Building..."
-            echo "  ${cargoBuild}"
             ${cargoBuild}
 
             runHook postBuild
@@ -176,8 +157,7 @@ with rec
           ''
             runHook preCheck
 
-            echo "Running test command:"
+            echo "Testing..."
-            echo "  ${cargoTest}"
             ${cargoTest}
 
             runHook postCheck
@@ -200,7 +180,7 @@ with rec
           cargoDoc="cargo doc --offline $doc_arg"
           echo "Running doc command:"
           echo "  $cargoDoc"
-          $cargoDoc
+          $cargoDoc || ${if doDocFail then "false" else "true" }
 
           ${lib.optionalString removeReferencesToSrcFromDocs ''
           # Remove references to the source derivation to reduce closure size
@@ -253,11 +233,7 @@ with rec
 
             runHook postInstall
           '';
-      } //
+      }
-      lib.optionalAttrs (! isNull cargolock )
-        { cargolock = builtinz.writeTOML "Cargo.lock" cargolock; } //
-      lib.optionalAttrs (! isNull cargotoml )
-        { cargotoml = builtinz.writeTOML "Cargo.toml" cargotoml; }
       )
       ;
 

default.nix

@@ -15,8 +15,8 @@
 with
   { libb = import ./lib.nix { inherit lib writeText runCommand remarshal; }; };
 
-with
+let
-  { defaultBuildAttrs =
+  defaultBuildAttrs =
       { inherit
           llvmPackages
           jq
@@ -30,65 +30,84 @@ with
           symlinkJoin
           cargo
           rustc;
-      };
+      }; in
-  };
 
-with
+let
-  { builtinz =
+  builtinz =
       builtins //
       import ./builtins
-        { inherit lib writeText remarshal runCommand ; };
+        { inherit lib writeText remarshal runCommand ; }; in
-  };
 
 # Crate building
 with rec
   {
       commonAttrs = src: attrs: rec
         { usePureFromTOML = attrs.usePureFromTOML or true;
-          cargolock = attrs.cargolock or null;
+          readTOML = builtinz.readTOML usePureFromTOML;
-          cargotoml = attrs.cargotoml or null;
-          cargolock' =
-            if isNull cargolock then
-              builtinz.readTOML usePureFromTOML "${src}/Cargo.lock"
-            else cargolock;
-          rootCargotoml =
-            if isNull cargotoml then
-              builtinz.readTOML usePureFromTOML "${src}/Cargo.toml"
-            else cargotoml;
 
-          # All the Cargo.tomls, including the top-level one
+          # The members we want to build
+          wantedMembers =
+            lib.mapAttrsToList (member: _cargotoml: member) wantedMemberCargotomls;
+
+          # Member path to cargotoml
+          wantedMemberCargotomls =
+            let pred =
+              if ! isWorkspace
+              then (_member: _cargotoml: true)
+              else
+                if builtins.hasAttr "targets" attrs
+                then (_member: cargotoml: lib.elem cargotoml.package.name attrs.targets)
+                else (member: _cargotoml: member != "."); in
+            lib.filterAttrs pred cargotomls;
+
+          # All cargotoml, from path to nix object
           cargotomls =
-            if builtins.hasAttr "package" rootCargotoml then
+            let readTOML = builtinz.readTOML usePureFromTOML; in
-              [rootCargotoml]
+
-            else
+            { "." = toplevelCargotoml; } //
-              with { members = rootCargotoml.workspace.members or []; };
+            lib.optionalAttrs isWorkspace
-              lib.filter (cargotoml:
+            (lib.listToAttrs
-                if builtins.hasAttr "targets" attrs then
+              (map
-                  lib.elem cargotoml.package.name attrs.targets
+                (member:
-                else true
+                  { name = member;
-              ) ( map
+                    value = readTOML (src + "/${member}/Cargo.toml");
-                (member: (builtinz.readTOML usePureFromTOML "${src}/${member}/Cargo.toml"))
+                  }
-                members );
+                )
+                (toplevelCargotoml.workspace.members or [])
+              )
+            );
+
+          # Are we building a workspace (or is this a simple crate) ?
+          isWorkspace = builtins.hasAttr "workspace" toplevelCargotoml;
+
+          # The top level Cargo.toml, either a workspace or package
+          toplevelCargotoml = readTOML (src + "/Cargo.toml");
+
+          # The cargo lock
+          cargolock = readTOML (src + "/Cargo.lock");
 
           # The list of paths to Cargo.tomls. If this is a workspace, the paths
           # are the members. Otherwise, there is a single path, ".".
-          cratePaths =
+          cratePaths = lib.concatStringsSep "\n" wantedMembers;
-            with rec
+          crateDependencies = libb.mkVersions cargolock;
-              { workspaceMembers = rootCargotoml.workspace.members or null;
-              };
-
-            if isNull workspaceMembers then "."
-            else
-              let pred = if builtins.hasAttr "targets" attrs then
-                (member: lib.elem member attrs.targets)
-                else (_member: true);
-              in lib.concatStringsSep "\n" (lib.filter pred workspaceMembers);
-          crateDependencies = libb.mkVersions cargolock';
           cargoBuild = attrs.cargoBuild or
+          ''
+            # Cargo uses mtime, and we write `src/main.rs` in the dep build
+            # step, so make sure cargo rebuilds stuff
+            find . -type f -name '*.rs' -exec touch {} +
+            for p in $cratePaths; do
+              pushd "$p"
+              echo "Building $p"
+              cargo build --$CARGO_BUILD_PROFILE -j $NIX_BUILD_CORES
+              popd
+            done
+          '';
+          cargoTest = attrs.cargoTest or
           ''
             for p in $cratePaths; do
               pushd "$p"
-              cargo build --$CARGO_BUILD_PROFILE -j $NIX_BUILD_CORES
+              echo "Running tests for $p"
+              cargo test --$CARGO_BUILD_PROFILE -j $NIX_BUILD_CORES
               popd
             done
           '';
@@ -97,93 +116,58 @@ with rec
         with (commonAttrs src attrs);
         import ./build.nix src
           ( defaultBuildAttrs //
-            { name =
+            { name = "some-name";
-                if lib.length cargotomls == 0 then
+              version = "some-version";
-                  abort "Found no cargotomls"
+              inherit cratePaths crateDependencies cargoBuild cargoTest;
-                else if lib.length cargotomls == 1 then
-                  (lib.head cargotomls).package.name
-                else
-                  "${(lib.head cargotomls).package.name}-and-others";
-              version = (lib.head cargotomls).package.version;
-              inherit cratePaths crateDependencies cargoBuild;
             } //
-            (removeAttrs attrs [ "targets" "usePureFromTOML" ])
+            (removeAttrs attrs [ "targets" "usePureFromTOML" "cargotomls" ])
           );
 
       buildPackageIncremental = src: attrs:
         with (commonAttrs src attrs);
-        with rec
+        import ./build.nix src
-          # FIXME: directDependencies should be built on a per-cargotoml basis.
+          (defaultBuildAttrs //
-          # All dependencies are not available in every member.
+            { name = "foo";
-          # Also, if a dependency is shared between two cargotomls, there's
+              version = "bar";
-          # (most of the time) no point recompiling it
+              inherit cratePaths crateDependencies cargoBuild cargoTest;
-          { buildDepsScript = writeText "prebuild-script"
+            } //
-              ''
+            (removeAttrs attrs [ "targets" "usePureFromTOML" "cargotomls" ]) //
-                cat ${builtinz.writeJSON "crates" ((directDependenciesList))} |\
+            { builtDependencies =
-                  jq -r \
+                [(
-                    --arg cbp $CARGO_BUILD_PROFILE \
+                  import ./build.nix
-                    --arg nbc $NIX_BUILD_CORES \
+                  (libb.dummySrc
-                    '.[] | "cargo build --\($cbp) -j \($nbc) -p \(.name):\(.version)"' |\
+                    { cargoconfig =
-                    while IFS= read -r c
+                        if builtinz.pathExists (src + "/.cargo/config")
-                    do
+                        then builtins.readFile (src + "/.cargo/config")
-                      echo "Running build command '$c'"
+                        else "";
-                      $c || echo "WARNING: one some dependencies failed to build: $c"
+                      cargolock = cargolock;
-                    done
+                      cargotomls = cargotomls;
-              '';
+                    }
-            isMember = name:
+                  )
-              lib.elem name (map (ctoml: ctoml.package.name) cargotomls);
+                  (defaultBuildAttrs //
-
+                    { name = "foo-deps";
-            isLocal = v:
+                      version = "bar";
-              ! builtins.hasAttr "path" v;
+                      inherit cratePaths crateDependencies ;
-
+                    } //
-            versions =
+                  (removeAttrs attrs [ "targets" "usePureFromTOML" "cargotomls" ]) //
-              lib.listToAttrs (
+                  { cargoBuild =
-              map (v: { name = v.name; value = v.version; })
+                      ''
-                crateDependencies);
+                        for p in $cratePaths; do
-
+                          pushd "$p"
-            directDependenciesList =
+                          cargo build --$CARGO_BUILD_PROFILE -j $NIX_BUILD_CORES || true
-              lib.filter (c:
+                          cargo test --$CARGO_BUILD_PROFILE -j $NIX_BUILD_CORES || true
-                builtins.hasAttr c.name directDependencies) crateDependencies;
+                          popd
-
+                        done
-            directDependencies =
+                      '';
-              lib.filterAttrs (_: v:
+                    cargoTest = "echo no tests for deps";
-                lib.isString v ||
+                    doCheck = false;
-                (! builtins.hasAttr "path" v)
+                    copyBuildArtifacts = true;
-                ) (
+                    copyDocsToSeparateOutput = false;
-              lib.foldr (x: y: x // y) {}
+                    name = "some-name";
-              (map (cargotoml:
+                  }
-                (lib.optionalAttrs (builtins.hasAttr "dependencies" cargotoml)
+                  )
-                  cargotoml.dependencies) //
+                )];
-                (lib.optionalAttrs (builtins.hasAttr "dev-dependencies" cargotoml)
+            });
-                  cargotoml.dev-dependencies)
-                ) cargotomls
-              ));
-          };
-        buildPackageSingleStep src
-          ((attrs) //
-          { builtDependencies =
-              [(
-              buildPackageSingleStep (libb.dummySrc src)
-                (attrs //
-                { cargoBuild = "source ${buildDepsScript}";
-                  doCheck = false;
-                  copyBuildArtifacts = true;
-                  copyDocsToSeparateOutput = false;
-                  cargolock = cargolock';
-                  cargotoml =
-                    { package = { name = "dummy"; version = "0.0.0"; }; } //
-                        { dependencies = directDependencies; }
-                    ;
-                name =
-                if lib.length cargotomls == 0 then
-                  abort "Found no cargotomls"
-                else if lib.length cargotomls == 1 then
-                  "${(lib.head cargotomls).package.name}-deps"
-                else
-                  "${(lib.head cargotomls).package.name}-and-others-deps";
-                })
-              )];
-          });
   };
 
 { inherit buildPackageSingleStep buildPackageIncremental crates;

lib.nix

@@ -82,20 +82,62 @@ rec
       };
 
     # A very minimal 'src' which makes cargo happy nonetheless
-    dummySrc = src:
+    dummySrc =
+      { cargoconfig   # string
+      , cargotomls   # attrset
+      , cargolock   # attrset
+      }:
       let
-        configContent =
+        config = writeText "config" cargoconfig;
-          if builtinz.pathExists "${src}/.cargo/config"
+        cargolock' = builtinz.writeTOML "Cargo.toml" cargolock;
-          then builtins.readFile "${src}/.cargo/config" else "";
+        cargotomlss = writeText "foo"
-        config = writeText "config" configContent;
+          (lib.concatStrings (lib.mapAttrsToList
+            (k: v: "${k}\n${builtinz.writeTOML "Cargo.toml-ds-aa" v}\n")
+            cargotomls
+          ));
+
       in
       runCommand "dummy-src" {}
       ''
         mkdir -p $out/.cargo
         cp -r ${config} $out/.cargo/config
+        cp ${cargolock'} $out/Cargo.lock
 
-        mkdir -p $out/src
+        cat ${cargotomlss} | \
-        touch $out/src/main.rs
+          while IFS= read -r member; do
+            read -r cargotoml
+            final_dir="$out/$member"
+            mkdir -p "$final_dir"
+            final_path="$final_dir/Cargo.toml"
+            cp $cargotoml "$final_path"
+
+            # make sure cargo is happy
+            pushd $out/$member > /dev/null
+
+            # TODO: get rid of all these hacks
+
+            sed -i '/.*=\s*{.*path\s*=.*/d' Cargo.toml
+            sed -i '/^build\s*=/d' Cargo.toml
+
+            mkdir -p src
+            touch src/main.rs
+            touch src/lib.rs
+            # Dirty hack to "touch" any bench, test, etc path.
+            cat Cargo.toml | \
+              grep -oP '^path\s*=\s*"\K\w+.rs' | \
+              while IFS= read -r path; do
+                mkdir -p $(dirname $path)
+                touch $path
+              done || true
+            cat Cargo.toml | \
+              grep -oP '^name\s*=\s*"\K\w+' | \
+              while IFS= read -r name; do
+                mkdir -p $name
+                mkdir -p benches
+                touch benches/$name.rs
+              done || true
+            popd > /dev/null
+          done
       '';
 
     mkPackages = cargolock:

test.nix

@@ -18,14 +18,14 @@ with
   { builtinz = builtins // pkgs.callPackage ./builtins {}; };
 
 rec
-{ rustfmt = naersk.buildPackage sources.rustfmt {};
+{ rustfmt = naersk.buildPackage sources.rustfmt { doDocFail = false; };
   rustfmt_test = pkgs.runCommand "rustfmt-test"
     { buildInputs = [ rustfmt ]; }
     "rustfmt --help && cargo-fmt --help && touch $out";
 
   ripgrep = naersk.buildPackage sources.ripgrep { usePureFromTOML = false; };
   # XXX: the `rg` executable is missing because we don't do `cargo install
-  # --path`.
+  # --path .`.
   # ripgrep_test = pkgs.runCommand "ripgrep-test"
   #   { buildInputs = [ ripgrep ]; }
   #   "rg --help && touch $out";