Merge pull request #4 from nmattia/nm-wip-3

Avoid IFD as much as possible

build.nix

@@ -17,8 +17,8 @@ src:
 , buildInputs ? []
 , nativeBuildInputs ? []
 , builtDependencies ? []
-, cargolockPath ? null
-, cargotomlPath ? null
+, cargolock ? null
+, cargotoml ? null
 , release ? true
 , stdenv
 , lib
@@ -37,23 +37,34 @@ src:
 with
   { builtinz =
       builtins //
-      import ./builtins.nix
+      import ./builtins
         { inherit lib writeText remarshal runCommand ; };
   };
 
 with rec
   {
-    drv = stdenv.mkDerivation
+    drv = stdenv.mkDerivation (
       { inherit
           src
           doCheck
           nativeBuildInputs
-          cargolockPath
-          cargotomlPath
           cratePaths
           name
           version;
 
+      cargoconfig = builtinz.toTOML
+        { source =
+            { crates-io = { replace-with = "nix-sources"; } ;
+              nix-sources =
+                { directory = symlinkJoin
+                    { name = "crates-io";
+                      paths = map (v: unpackCrate v.name v.version v.sha256)
+                        crateDependencies;
+                    };
+                };
+            };
+        };
+
         outputs = [ "out" ] ++ lib.optional doDoc "doc";
         preInstallPhases = lib.optional doDoc [ "docPhase" ];
 
@@ -88,24 +99,24 @@ with rec
           ''
             runHook preConfigure
 
-            if [ -n "$cargolockPath" ]
+            if [ -n "$cargolock" ]
             then
               echo "Setting Cargo.lock"
               if [ -f "Cargo.lock" ]
               then
                 echo "WARNING: replacing existing Cargo.lock"
               fi
-              cp --no-preserve mode "$cargolockPath" Cargo.lock
+              echo "$cargolock" > Cargo.lock
             fi
 
-            if [ -n "$cargotomlPath" ]
+            if [ -n "$cargotoml" ]
             then
               echo "Setting Cargo.toml"
               if [ -f "Cargo.toml" ]
               then
                 echo "WARNING: replacing existing Cargo.toml"
               fi
-              cp "$cargotomlPath" Cargo.toml
+              echo "$cargotoml" > Cargo.toml
             fi
 
             mkdir -p target
@@ -127,7 +138,7 @@ with rec
             export CARGO_HOME=''${CARGO_HOME:-$PWD/.cargo-home}
             mkdir -p $CARGO_HOME
 
-            cp --no-preserve mode ${cargoconfig} $CARGO_HOME/config
+            echo "$cargoconfig" > $CARGO_HOME/config
 
             # TODO: figure out why "1" works whereas "0" doesn't
             find . -type f -exec touch --date=@1 {} +
@@ -209,7 +220,13 @@ with rec
 
             runHook postInstall
           '';
-      };
+      } //
+      lib.optionalAttrs (! isNull cargolock )
+        { cargolock = builtinz.toTOML cargolock; } //
+      lib.optionalAttrs (! isNull cargotoml )
+        { cargotoml = builtinz.toTOML cargotoml; }
+      )
+      ;
 
     # XXX: the actual crate format is not documented but in practice is a
     # gzipped tar; we simply unpack it and introduce a ".cargo-checksum.json"
@@ -227,18 +244,5 @@ with rec
         tar -xvzf ${crate} -C $out
         echo '{"package":"${sha256}","files":{}}' > $out/${name}-${version}/.cargo-checksum.json
       '';
-
-    cargoconfig = builtinz.writeTOML
-      { source =
-          { crates-io = { replace-with = "nix-sources"; } ;
-            nix-sources =
-              { directory = symlinkJoin
-                  { name = "crates-io";
-                    paths = map (v: unpackCrate v.name v.version v.sha256)
-                      crateDependencies;
-                  };
-              };
-          };
-      };
   };
 if isNull override then drv else drv.overrideAttrs override

builtins/default.nix

@@ -1,17 +1,27 @@
 # some extra "builtins"
-{ lib, writeText, runCommand, remarshal }:
+{ lib
+, writeText
+, runCommand
+, remarshal
+}:
 
+rec
 {
-    # Nix < 2.3 cannot parse all TOML files
-    # https://github.com/NixOS/nix/issues/2901
-    # can then be replaced with:
-    #   readTOML = f: builtins.fromTOML (builtins.readFile f);
-    readTOML = f: builtins.fromJSON (builtins.readFile (runCommand "read-toml"
+    toTOML = import ./to-toml.nix { inherit lib; };
+    writeTOML = attrs: writeText "write-toml" (toTOML attrs);
+
+    readTOML = usePure: f:
+      if usePure then
+        builtins.fromTOML (builtins.readFile f)
+      else
+        builtins.fromJSON (builtins.readFile (
+          runCommand "from-toml"
           { buildInputs = [ remarshal ];
             allowSubstitutes = false;
             preferLocalBuild = true;
           }
           ''
+            echo "$from_toml_in" > in.toml
             remarshal \
               -if toml \
               -i ${f} \
@@ -19,19 +29,6 @@
               -o $out
           ''));
 
-    writeTOML = attrs: runCommand "write-toml"
-      { buildInputs = [ remarshal ];
-        allowSubstitutes = false;
-        preferLocalBuild = true;
-      }
-      ''
-        remarshal \
-          -if json \
-          -i ${writeText "toml-json" (builtins.toJSON attrs)} \
-          -of toml \
-          -o $out
-      '';
-
     writeJSON = name: attrs: writeText name
       (builtins.toJSON attrs);
 

builtins/to-toml.nix

@@ -0,0 +1,130 @@
+{ lib }:
+let
+  inherit (lib)
+    length
+    elemAt
+    concatMap
+    all
+    concatLists
+    concatStringsSep
+    concatMapStringsSep
+    mapAttrsToList
+    ;
+  inherit (builtins)
+    abort
+    match
+    toJSON
+    typeOf
+    ;
+
+  quoteKey = k:
+    if match "[a-zA-Z]+" k == []
+    then k
+    else quoteString k;
+
+  quoteString = builtins.toJSON;
+
+  outputValInner = v:
+    let ty = tomlTy v; in
+    if ty == "set" then
+      let
+        vals = mapAttrsToList
+          (k': v': "${quoteKey k'} = ${outputValInner v'}") v;
+        valsStr = concatStringsSep ", " vals;
+      in
+      "{ ${valsStr} }" else
+    outputVal v;
+
+  outputVal = v:
+    let ty = tomlTy v; in
+    if ty == "bool" then
+      builtins.toJSON v
+      else
+    if ty == "string" then
+      quoteString v
+      else
+    if ty == "list" || ty == "list_of_attrs" then
+      let
+        vals = map quoteString v;
+        valsStr = concatStringsSep ", " vals;
+        in
+      "[ ${valsStr} ]" else
+    if ty == "set" then
+      abort "unsupported set for not-inner value"
+    else abort "Not implemented: type ${ty}"
+      ;
+
+  outputKeyValInner = k: v:
+    let ty = tomlTy v; in
+    if ty == "set" then
+      let
+        vals = mapAttrsToList
+          (k': v': "${quoteKey k'} = ${outputValInner v'}") v;
+        valsStr = concatStringsSep ", " vals;
+      in
+      [ "${quoteKey k} = { ${valsStr} }" ] else
+    outputKeyVal k v;
+
+  # Returns a list of strings; one string per line
+  outputKeyVal = k: v:
+    let ty = tomlTy v; in
+    if ty == "bool" then
+      [ "${quoteKey k} = ${outputValInner v}" ]
+      else
+    if ty == "string" then
+      [ "${quoteKey k} = ${quoteString v}" ]
+      else
+    if ty == "list_of_attrs" then
+      concatMap (inner:
+      [ "[[${k}]]" ] ++
+      (concatLists (mapAttrsToList outputKeyValInner inner))) v
+      else
+    if ty == "list" then
+      let
+        vals = map quoteString v;
+        valsStr = concatStringsSep ", " vals;
+        in
+      [ "${quoteKey k} = [ ${valsStr} ]" ] else
+    if ty == "set" then
+      [ "[${k}]" ] ++
+      (concatLists (mapAttrsToList outputKeyValInner v))
+    else abort "Not implemented: type ${ty} for key ${k}"
+      ;
+
+  tomlTy = x:
+    if typeOf x == "string" then "string" else
+    if typeOf x == "bool" then "bool" else
+    if typeOf x == "int" then "int" else
+    if typeOf x == "float" then "float" else
+    if typeOf x == "set" then
+      if lib.isDerivation x then "string" else "set" else
+    if typeOf x == "list" then
+      if length x == 0 then "list"
+      else
+        let ty = typeOf (elemAt x 0); in
+        #assert (all (v: typeOf v == ty) x);
+        if ty == "set" then "list_of_attrs" else "list"
+    else abort "Not implemented: toml type for ${typeOf x}";
+
+  toTOML = attrs:
+    assert (typeOf attrs == "set");
+    let
+      byTy = lib.foldl
+        (acc: x:
+          let ty = tomlTy x.v; in
+          acc // { "${ty}" = (acc.${ty} or []) ++ [ x ]; }
+        )
+        {} (mapAttrsToList (k: v: { inherit k v ; }) attrs);
+    in
+    concatMapStringsSep "\n"
+      (kv: concatStringsSep "\n" (outputKeyVal kv.k kv.v))
+      ( (byTy.string or []) ++
+        (byTy.int or []) ++
+        (byTy.float or []) ++
+        (byTy.list or []) ++
+        (byTy.list_of_attrs or []) ++
+        (byTy.set or [])
+      )
+        ;
+in
+  toTOML

default.nix

@@ -36,7 +36,7 @@ with
 with
   { builtinz =
       builtins //
-      import ./builtins.nix
+      import ./builtins
         { inherit lib writeText remarshal runCommand ; };
   };
 
@@ -44,18 +44,17 @@ with
 with rec
   {
       commonAttrs = src: attrs: rec
-        { cargolockPath = attrs.cargolockPath or null;
-          cargotomlPath = attrs.cargotomlPath or null;
-          cargolock =
-            if isNull cargolockPath then
-              builtinz.readTOML "${src}/Cargo.lock"
-            else
-              builtinz.readTOML cargolockPath;
+        { usePureFromTOML = attrs.usePureFromTOML or true;
+          cargolock = attrs.cargolock or null;
+          cargotoml = attrs.cargotoml or null;
+          cargolock' =
+            if isNull cargolock then
+              builtinz.readTOML usePureFromTOML "${src}/Cargo.lock"
+            else cargolock;
           rootCargotoml =
-            if isNull cargotomlPath then
-              builtinz.readTOML "${src}/Cargo.toml"
-            else
-              builtinz.readTOML cargotomlPath;
+            if isNull cargotoml then
+              builtinz.readTOML usePureFromTOML "${src}/Cargo.toml"
+            else cargotoml;
 
           # All the Cargo.tomls, including the top-level one
           cargotomls =
@@ -68,7 +67,7 @@ with rec
                   lib.elem cargotoml.package.name attrs.targets
                 else true
               ) ( map
-                (member: (builtinz.readTOML "${src}/${member}/Cargo.toml"))
+                (member: (builtinz.readTOML usePureFromTOML "${src}/${member}/Cargo.toml"))
                 members );
 
           # The list of paths to Cargo.tomls. If this is a workspace, the paths
@@ -80,7 +79,7 @@ with rec
 
             if isNull workspaceMembers then "."
             else lib.concatStringsSep "\n" workspaceMembers;
-          crateDependencies = libb.mkVersions cargolock;
+          crateDependencies = libb.mkVersions cargolock';
           targetInstructions =
             if builtins.hasAttr "targets" attrs then
               lib.concatMapStringsSep " " (target: "-p ${target}") attrs.targets
@@ -102,7 +101,7 @@ with rec
               version = (lib.head cargotomls).package.version;
               inherit cratePaths crateDependencies cargoBuild;
             } //
-            (removeAttrs attrs [ "targets"])
+            (removeAttrs attrs [ "targets" "usePureFromTOML" ])
           );
 
       buildPackageIncremental = src: attrs:
@@ -163,12 +162,10 @@ with rec
                 { cargoBuild = "source ${buildDepsScript}";
                   doCheck = false;
                   copyBuildArtifacts = true;
-                  cargolockPath = builtinz.writeTOML cargolock;
-                  cargotomlPath = builtinz.writeTOML
-                    (
+                  cargolock = cargolock';
+                  cargotoml =
                     { package = { name = "dummy"; version = "0.0.0"; }; } //
                         { dependencies = directDependencies; }
-                    )
                     ;
                 name =
                 if lib.length cargotomls == 0 then

lib.nix

@@ -2,7 +2,7 @@
 with
   { builtinz =
       builtins //
-      import ./builtins.nix
+      import ./builtins
         { inherit lib writeText remarshal runCommand ; };
   };
 rec

test.nix

@@ -15,7 +15,7 @@ with rec
   };
 
 with
-  { builtinz = builtins // pkgs.callPackage ./builtins.nix {}; };
+  { builtinz = builtins // pkgs.callPackage ./builtins {}; };
 
 rec
 { rustfmt = naersk.buildPackage sources.rustfmt {};
@@ -23,7 +23,7 @@ rec
     { buildInputs = [ rustfmt ]; }
     "rustfmt --help && cargo-fmt --help && touch $out";
 
-  ripgrep = naersk.buildPackage sources.ripgrep {};
+  ripgrep = naersk.buildPackage sources.ripgrep { usePureFromTOML = false; };
   # XXX: executables are missing
   #ripgrep_test = pkgs.runCommand "ripgrep-test"
     #{ buildInputs = [ ripgrep ]; }
@@ -98,7 +98,8 @@ rec
   cargo =
     with rec
       { cargoSrc = sources.cargo;
-        cargoCargoToml = builtinz.readTOML "${cargoSrc}/Cargo.toml";
+        # cannot use the pure readTOML
+        cargoCargoToml = builtinz.readTOML false "${cargoSrc}/Cargo.toml";
 
         # XXX: this works around some hack that breaks the build. For more info
         # on the hack, see
@@ -109,11 +110,11 @@ rec
               cargoCargoToml.dependencies;
           };
 
-        cargoCargoLock = "${sources.rust}/Cargo.lock";
+        cargoCargoLock = builtinz.readTOML true "${sources.rust}/Cargo.lock";
       };
     naersk.buildPackage cargoSrc
-      { cargolockPath = cargoCargoLock;
-        cargotomlPath = builtinz.writeTOML cargoCargoToml';
+      { cargolock = cargoCargoLock;
+        cargotoml = cargoCargoToml';
 
         # Tests fail, although cargo seems to operate normally
         doCheck = false;