Merge pull request #203 from nickel-lang/transparent-regen

Make the file generation more transparent

doc/filegen.md

@@ -29,7 +29,10 @@ This has two main advantages:
 
 ## Usage
 
-To re-generate the files run:
+The files specified in `files.*` are regenerated automatically when entering
+the devshell (unless `filegen_hook` is set to false).
+
+They can also be regenerated manually with:
 
 ```bash
 nix run .#regenerate-files

lib/files.ncl

@@ -11,7 +11,18 @@ let File = {
     | doc m%"
         The content of the file.
       "%
-    | nix.derivation.NixString,
+    | nix.derivation.NullOr nix.derivation.NixString
+    | default
+    = null,
+  file
+    | doc "File from which to read the body of the script"
+    | nix.derivation.NullOr nix.derivation.NixString
+    | default
+    =
+      if content == null then
+        null
+      else
+        nix.builtins.to_file "generated-content" content,
   materialisation_method
     : [| 'Symlink, 'Copy |]
     | doc m%"
@@ -35,7 +46,6 @@ let regenerate_files | Files -> nix.derivation.Derivation
   = fun files_to_generate =>
     let regnerate_one | String -> File -> nix.derivation.NixString
       = fun key file_descr =>
-        let file_content = file_descr.content in
         let target = file_descr.target in
         let copy_command =
           match {
@@ -44,17 +54,17 @@ let regenerate_files | Files -> nix.derivation.Derivation
           }
             file_descr.materialisation_method
         in
-        let file_in_store =
-          nix.builtins.to_file
-            (nix.utils.escape_drv_name key)
-            file_content
-        in
         nix-s%"
-            rm -f %{target}
-            echo "Regenerating %{target}"
-	    target_dir=$(dirname %{target})
-	    test "${target_dir}" != "." && mkdir -p "${target_dir}"
-            %{copy_command} %{file_in_store} %{target}
+            if [[ ! -f "%{target}" ]] || [[ $(cat "%{target}") != $(cat "%{file_descr.file}") ]]; then
+              rm -f %{target}
+              echo "Regenerating %{target}"
+              target_dir=$(dirname "%{target}")
+              test "${target_dir}" != "." && mkdir -p "${target_dir}"
+              # XXX: If `source.file` is set explicitely to a relative path
+              # and `materialisation_method` is `'Symlink`, this will link to the
+              # original file, not one in the store. Not sure that's what we want.
+              %{copy_command} "%{file_descr.file}" "%{target}"
+            fi
           "%
       in
     {
@@ -84,11 +94,32 @@ let regenerate_files | Files -> nix.derivation.Derivation
         Set of files that should be generated in the project's directory.
       "%
       = {},
+    filegen_hook.enable
+      | Bool
+      | doc m%"
+        Enable a hook that will automatically regenerate the files managed by
+        Nickel when entering the shell.
+      "%
+      | default
+      = true,
     flake.apps, # Forward declaration. Not great but would need some refactor to fix
+    shells,
   },
   config | Schema
     = {
       files,
+      filegen_hook,
+      shells,
       flake.apps.regenerate-files.program = nix-s%"%{regenerate_files files}/bin/regenerate-files"%,
+
+      shells.build.hooks =
+        if filegen_hook.enable then
+          {
+            filegen_hook = nix-s%"
+              %{regenerate_files files}/bin/regenerate-files
+            "%
+          }
+        else
+          {},
     },
 }

lib/lib.nix

@@ -169,23 +169,19 @@
       passAsFile = ["expectedLockfileContents"];
     } (
       if needNewLockfile
-      then
-        lib.warn ''
-          Lockfile contents are outdated. Please run "nix run .#regenerate-lockfile" to update them.
-        ''
-        ''
-          cp -r "${sources}" sources
-          if [ -f sources/nickel.lock.ncl ]; then
-            chmod +w sources sources/nickel.lock.ncl
-          else
-            chmod +w sources
-          fi
-          cp $expectedLockfileContentsPath sources/nickel.lock.ncl
-          cat > eval.ncl <<EOF
-          ${nickelWithImports "sources"}
-          EOF
-          ${nickel}/bin/nickel export eval.ncl --field config.flake > $out
-        ''
+      then ''
+        cp -r "${sources}" sources
+        if [ -f sources/nickel.lock.ncl ]; then
+          chmod +w sources sources/nickel.lock.ncl
+        else
+          chmod +w sources
+        fi
+        cp $expectedLockfileContentsPath sources/nickel.lock.ncl
+        cat > eval.ncl <<EOF
+        ${nickelWithImports "sources"}
+        EOF
+        ${nickel}/bin/nickel export eval.ncl --field config.flake > $out
+      ''
       else ''
         cat > eval.ncl <<EOF
         ${nickelWithImports sources}
@@ -212,9 +208,18 @@
     nickelResult = callNickel {
       inherit nickelFile baseDir flakeInputs lockFileContents;
     };
+    enrichedFlakeInputs =
+      flakeInputs
+      // {
+        # Dummy “flake input” used to pass some data through the Nickel
+        # evaluation.
+        # We should ideally get rid of that and find another more principled
+        # way, but that will do for now.
+        "%%organist_internal".nickelLock = builtins.toFile "nickel.lock.ncl" (buildLockFileContents lockFileContents);
+      };
   in
     {rawNickel = nickelResult;}
-    // (importFromNickel flakeInputs system baseDir (builtins.fromJSON
+    // (importFromNickel enrichedFlakeInputs system baseDir (builtins.fromJSON
         (builtins.unsafeDiscardStringContext (builtins.readFile nickelResult))));
 in {
   inherit

lib/lockfile.ncl

@@ -0,0 +1,9 @@
+let filegen = import "files.ncl" in
+let nix = import "nix-interop/nix.ncl" in
+{
+  Schema = filegen.Schema,
+  config | Schema
+    = {
+      files."nickel.lock.ncl".file = nix.import_nix "%%organist_internal#nickelLock",
+    }
+}

lib/schema.ncl

@@ -1,5 +1,6 @@
 let nix = import "./nix-interop/nix.ncl" in
 let filegen = import "files.ncl" in
+let lockfile = import "lockfile.ncl" in
 {
   OrganistShells = {
     dev
@@ -71,4 +72,5 @@ let filegen = import "files.ncl" in
         },
     }
     & filegen
+    & lockfile
 }

project.ncl

@@ -72,6 +72,27 @@ organist.OrganistExpression
           touch $out
         "%,
           },
+
+          lsp = {
+            name = "organist-lsp-integration",
+            version = "0.0",
+            env.buildInputs = {
+              nls = import_nix "nixpkgs#nls",
+              python3 = import_nix "nixpkgs#python3",
+              pygls = import_nix "nixpkgs#python3Packages.pygls",
+              pytest = import_nix "nixpkgs#python3Packages.pytest",
+              pytest-asyncio = import_nix "nixpkgs#python3Packages.pytest-asyncio",
+            },
+            env = {
+              src = import_nix "self",
+              phases = ["unpackPhase", "testPhase", "installPhase"],
+              testPhase = nix-s%"
+                  cd tests/lsp
+                  pytest | tee $out
+                "%,
+              installPhase = "touch $out",
+            },
+          },
         },
 
       flake.checks = import "tests/main.ncl",

run-test.sh

@@ -51,28 +51,24 @@ test_one_template () (
   sed -i "s/shells\.Bash/shells.$target/" project.ncl
   prepare_shell
 
-  STORED_LOCKFILE_CONTENTS="$(cat nickel.lock.ncl)"
   TEST_SCRIPT="$(nickel export --format raw <<<'(import "'"$PROJECT_ROOT"'/lib/shell-tests.ncl").'"$target"'.script')"
 
   echo "Running with incorrect nickel.lock.ncl" 1>&2
+  echo '{}' > nickel.lock.ncl
   nix develop --accept-flake-config --print-build-logs --command bash <<<"$TEST_SCRIPT"
 
   if [[ $isFull == false ]]; then
     return
   fi
 
   echo "Running without nickel.lock.ncl" 1>&2
-  rm nickel.lock.ncl
+  rm -f nickel.lock.ncl
   nix develop --accept-flake-config --print-build-logs --command bash <<<"$TEST_SCRIPT"
 
   echo "Run with proper nickel.lock.ncl" 1>&2
-  nix run .\#regenerate-lockfile
-  PROPER_LOCKFILE_CONTENTS="$(cat nickel.lock.ncl)"
   nix develop --accept-flake-config --print-build-logs --command bash <<<"$TEST_SCRIPT"
 
   echo "Testing without flakes" 1>&2
-  # restore lockfile
-  cat > nickel.lock.ncl <<<"$STORED_LOCKFILE_CONTENTS"
   # pretend it's not flake anymore
   rm flake.*
   cat > shell.nix <<EOF
@@ -86,14 +82,15 @@ in
 EOF
 
   echo "Running with incorrect nickel.lock.ncl" 1>&2
+  rm -f nickel.lock.ncl
+  echo '{}' > nickel.lock.ncl
   nix develop --impure -f shell.nix -I nixpkgs="$NIXPKGS_PATH" --command bash <<<"$TEST_SCRIPT"
 
   echo "Running without nickel.lock.ncl" 1>&2
-  rm nickel.lock.ncl
+  rm -f nickel.lock.ncl
   nix develop --impure -f shell.nix -I nixpkgs="$NIXPKGS_PATH" --command bash <<<"$TEST_SCRIPT"
 
   echo "Run with proper nickel.lock.ncl" 1>&2
-  cat > nickel.lock.ncl <<<"$PROPER_LOCKFILE_CONTENTS"
   nix develop --impure -f shell.nix -I nixpkgs="$NIXPKGS_PATH" --command bash <<<"$TEST_SCRIPT"
 
   popd
@@ -120,7 +117,6 @@ test_example () (
   cp -r "$examplePath" ./example
   pushd ./example
   prepare_shell
-  nix run .\#regenerate-files --print-build-logs
   nix develop --print-build-logs --command bash test.sh
   popd
   popd

tests/lsp/.gitignore

@@ -0,0 +1 @@
+__pycache__

tests/lsp/conftest.py

@@ -0,0 +1,21 @@
+import testlib
+import asyncio
+import pytest
+import pytest_asyncio
+from lsprotocol import types as lsp
+
+@pytest_asyncio.fixture
+async def client():
+    # Setup
+    client = testlib.LanguageClient("organist-test-suite", "v1")
+    await client.start_io("nls")
+    response = await client.initialize_async(
+            lsp.InitializeParams(
+                capabilities=lsp.ClientCapabilities(),
+                root_uri="."
+            )
+        )
+    assert response is not None
+    client.initialized(lsp.InitializedParams())
+    return client
+

tests/lsp/template/nickel.lock.ncl

@@ -0,0 +1 @@
+{ organist = import "../../../lib/organist.ncl" }