release: forbid use of lib.fileset in Nixpkgs (#369694)

doc/doc-support/lib-function-docs.nix

@@ -94,10 +94,7 @@
 stdenvNoCC.mkDerivation {
   name = "nixpkgs-lib-docs";
 
-  src = lib.fileset.toSource {
-    root = ../..;
-    fileset = ../../lib;
-  };
+  src = ../../lib;
 
   nativeBuildInputs = [
     nixdoc
@@ -105,6 +102,8 @@ stdenvNoCC.mkDerivation {
   ];
 
   installPhase = ''
+    cd ..
+
     export NIX_STATE_DIR=$(mktemp -d)
     nix-instantiate --eval --strict --json ${./lib-function-locations.nix} \
       --arg nixpkgsPath "./." \

doc/doc-support/package.nix

@@ -25,16 +25,22 @@ stdenvNoCC.mkDerivation (
 
     nativeBuildInputs = [ nixos-render-docs ];
 
-    src = lib.fileset.toSource {
-      root = ../.;
-      fileset = lib.fileset.unions [
-        (lib.fileset.fileFilter (file: file.hasExt "md" || file.hasExt "md.in") ../.)
-        ../style.css
-        ../anchor-use.js
-        ../anchor.min.js
-        ../manpage-urls.json
-        ../redirects.json
-      ];
+    src = lib.cleanSourceWith {
+      src = ../.;
+      filter =
+        path: type:
+        type == "directory"
+        || lib.hasSuffix ".md" path
+        || lib.hasSuffix ".md.in" path
+        || lib.elem path (
+          map toString [
+            ../style.css
+            ../anchor-use.js
+            ../anchor.min.js
+            ../manpage-urls.json
+            ../redirects.json
+          ]
+        );
     };
 
     postPatch = ''

nixos/lib/test-driver/default.nix

@@ -14,22 +14,12 @@
   extraPythonPackages ? (_: [ ]),
   nixosTests,
 }:
-let
-  fs = lib.fileset;
-in
 python3Packages.buildPythonApplication {
   pname = "nixos-test-driver";
   version = "1.1";
   pyproject = true;
 
-  src = fs.toSource {
-    root = ./.;
-    fileset = fs.unions [
-      ./pyproject.toml
-      ./test_driver
-      ./extract-docstrings.py
-    ];
-  };
+  src = ./src;
 
   build-system = with python3Packages; [
     setuptools

nixos/lib/test-driver/nixos-test-driver-docstrings.nix

@@ -11,6 +11,6 @@ in
 
 runCommand "nixos-test-driver-docstrings" env ''
   mkdir $out
-  python3 ${./extract-docstrings.py} ${./test_driver/machine.py} \
+  python3 ${./src/extract-docstrings.py} ${./src/test_driver/machine.py} \
     > $out/machine-methods.md
 ''

pkgs/build-support/testers/shellcheck/tester.nix

@@ -2,6 +2,7 @@
 {
   lib,
   stdenv,
+  runCommand,
   shellcheck,
 }:
 
@@ -10,7 +11,7 @@
 # Tests: ./tests.nix
 { src }:
 let
-  inherit (lib) fileset pathType isPath;
+  inherit (lib) pathType isPath;
 in
 stdenv.mkDerivation {
   name = "run-shellcheck";
@@ -18,10 +19,10 @@ stdenv.mkDerivation {
     if
       isPath src && pathType src == "regular" # note that for strings this would have been IFD, which we prefer to avoid
     then
-      fileset.toSource {
-        root = dirOf src;
-        fileset = src;
-      }
+      runCommand "testers-shellcheck-src" { } ''
+        mkdir $out
+        cp ${src} $out
+      ''
     else
       src;
   nativeBuildInputs = [ shellcheck ];

pkgs/build-support/testers/shellcheck/tests.nix

@@ -6,9 +6,6 @@
   testers,
   runCommand,
 }:
-let
-  inherit (lib) fileset;
-in
 lib.recurseIntoAttrs {
 
   example-dir =
@@ -16,12 +13,7 @@ lib.recurseIntoAttrs {
       {
         failure = testers.testBuildFailure (
           testers.shellcheck {
-            src = fileset.toSource {
-              root = ./.;
-              fileset = fileset.unions [
-                ./example.sh
-              ];
-            };
+            src = ./src;
           }
         );
       }
@@ -37,7 +29,7 @@ lib.recurseIntoAttrs {
       {
         failure = testers.testBuildFailure (
           testers.shellcheck {
-            src = ./example.sh;
+            src = ./src/example.sh;
           }
         );
       }

pkgs/by-name/fl/flattenReferencesGraph/package.nix

@@ -1,25 +1,19 @@
 {
   callPackage,
-  lib,
+  nix-gitignore,
   python3Packages,
 }:
 let
-  inherit (lib) fileset;
   helpers = callPackage ./helpers.nix { };
   pythonPackages = python3Packages;
+
 in
 pythonPackages.buildPythonApplication {
   version = "0.1.0";
   pname = "flatten-references-graph";
 
-  src = fileset.toSource {
-    root = ./src;
-    fileset = fileset.unions [
-      ./src/.flake8
-      ./src/flatten_references_graph
-      ./src/setup.py
-    ];
-  };
+  # Note: this uses only ./src/.gitignore
+  src = nix-gitignore.gitignoreSource [ ] ./src;
 
   propagatedBuildInputs = with pythonPackages; [
     igraph

pkgs/by-name/pu/purescm/README.rst

@@ -14,6 +14,6 @@ Suggested additional ``buildInputs``
 To update this package
 ======================
 
-#. Bump the ``./package.json`` version pin
-#. Run ``nix-shell -p nodejs --command "npm i --package-lock-only"``
+#. Bump the ``./manifests/package.json`` version pin
+#. Run ``(cd manifests && nix-shell -p nodejs --command "npm i --package-lock-only")``
 #. Update ``npmDeps.hash`` in the ``package.nix``

pkgs/by-name/pu/purescm/package.nix

@@ -8,7 +8,7 @@
 let
   inherit (lib) fileset;
 
-  packageLock = builtins.fromJSON (builtins.readFile ./package-lock.json);
+  packageLock = builtins.fromJSON (builtins.readFile ./manifests/package-lock.json);
 
   pname = "purescm";
   version = packageLock.packages."node_modules/${pname}".version;
@@ -16,24 +16,11 @@ let
   package = buildNpmPackage {
     inherit pname version;
 
-    src = fileset.toSource {
-      root = ./.;
-      fileset = fileset.unions [
-        ./package.json
-        ./package-lock.json
-        ./.gitignore
-      ];
-    };
+    src = ./manifests;
     dontNpmBuild = true;
 
     npmDeps = fetchNpmDeps {
-      src = fileset.toSource {
-        root = ./.;
-        fileset = fileset.unions [
-          ./package-lock.json
-          ./package.json
-        ];
-      };
+      src = ./manifests;
       hash = "sha256-ljeFcLvIET77Q0OR6O5Ok1fGnaxaKaoywpcy2aHq/6o=";
     };
 

pkgs/by-name/sh/shopify-cli/package.nix

@@ -6,13 +6,7 @@ buildNpmPackage {
   pname = "shopify";
   version = version;
 
-  src = lib.fileset.toSource {
-    root = ./.;
-    fileset = with lib.fileset; unions [
-      ./package.json
-      ./package-lock.json
-    ];
-  };
+  src = ./manifests;
 
   npmDepsHash = "sha256-QhbOKOs/0GEOeySG4uROzgtD4o7C+6tS/TAaPcmC3xk=";
   dontNpmBuild = true;

pkgs/by-name/sh/shopify-cli/update.sh

@@ -18,6 +18,8 @@ if [[ "$UPDATE_NIX_OLD_VERSION" == "$version" ]]; then
     exit 0
 fi
 
+pushd manifests
+
 # Update the package.json
 sed -i "s|$UPDATE_NIX_OLD_VERSION|$version|g" package.json
 
@@ -25,6 +27,9 @@ sed -i "s|$UPDATE_NIX_OLD_VERSION|$version|g" package.json
 rm -f package-lock.json
 npm i --package-lock-only
 npm_hash=$(prefetch-npm-deps package-lock.json)
+
+popd
+
 sed -i "s|npmDepsHash = \".*\";|npmDepsHash = \"$npm_hash\";|" package.nix
 
 popd

pkgs/development/cuda-modules/saxpy/default.nix

@@ -18,19 +18,12 @@ let
     libcublas
     ;
   inherit (lib) getDev getLib getOutput;
-  fs = lib.fileset;
 in
 backendStdenv.mkDerivation {
   pname = "saxpy";
   version = "unstable-2023-07-11";
 
-  src = fs.toSource {
-    root = ./.;
-    fileset = fs.unions [
-      ./CMakeLists.txt
-      ./saxpy.cu
-    ];
-  };
+  src = ./src;
 
   __structuredAttrs = true;
   strictDeps = true;

pkgs/development/python-modules/waitress-django/default.nix

@@ -6,22 +6,12 @@
   waitress,
 }:
 
-let
-  fs = lib.fileset;
-in
-
 buildPythonPackage {
   pname = "waitress-django";
   version = "1.0.0";
   pyproject = true;
 
-  src = fs.toSource {
-    root = ./.;
-    fileset = fs.unions [
-      ./setup.py
-      ./src
-    ];
-  };
+  src = ./src;
 
   build-system = [ setuptools ];
 

pkgs/development/python-modules/waitress-django/src/setup.py

@@ -7,6 +7,5 @@ setup( name         = "waitress-django"
      , description  = "A waitress WSGI server serving django"
      , author       = "Bas van Dijk"
      , author_email = "v.dijk.bas@gmail.com"
-     , package_dir  = {"" : "src"}
-     , scripts      = ["src/waitress-serve-django"]
+     , scripts      = ["waitress-serve-django"]
      )

pkgs/development/tools/yarn2nix-moretea/default.nix

@@ -398,15 +398,7 @@ in rec {
     });
 
   yarn2nix = mkYarnPackage {
-    src = lib.fileset.toSource {
-      root = ./.;
-      fileset = lib.fileset.unions [
-        ./bin
-        ./lib
-        ./package.json
-        ./yarn.lock
-      ];
-    };
+    src = ./yarn2nix;
 
     # yarn2nix is the only package that requires the yarnNix option.
     # All the other projects can auto-generate that file.
@@ -415,7 +407,7 @@ in rec {
     # Using the filter above and importing package.json from the filtered
     # source results in an error in restricted mode. To circumvent this,
     # we import package.json from the unfiltered source
-    packageJSON = ./package.json;
+    packageJSON = ./yarn2nix/package.json;
 
     yarnFlags = defaultYarnFlags ++ [ "--ignore-scripts" "--production=true" ];
 
@@ -446,7 +438,7 @@ in rec {
     mkdir -p $out/lib
     mkdir -p $out/bin
 
-    cp ${./lib/urlToName.js} $out/lib/urlToName.js
+    cp ${./yarn2nix/lib/urlToName.js} $out/lib/urlToName.js
     cp ${./internal/fixup_yarn_lock.js} $out/bin/fixup_yarn_lock
 
     patchShebangs $out

pkgs/test/haskell/cabalSdist/default.nix

@@ -6,18 +6,7 @@
 }:
 
 let
-  src = lib.fileset.toSource {
-    root = ./local;
-    fileset = lib.fileset.unions [
-      ./local/app
-      ./local/CHANGELOG.md
-      ./local/local.cabal
-    ];
-  };
-  # This prevents the source from depending on the formatting of the ./local/generated.nix file
-  localRaw = haskell.lib.compose.overrideSrc {
-    inherit src;
-  } (haskellPackages.callPackage ./local/generated.nix { });
+  localRaw = haskellPackages.callPackage ./generated.nix { };
 in
 lib.recurseIntoAttrs rec {
 
@@ -28,7 +17,6 @@ lib.recurseIntoAttrs rec {
 
   localFromCabalSdist = haskellPackages.buildFromCabalSdist localRaw;
 
-  # NOTE: ./local refers to the "./." path in `./local/generated.nix`.
   # This test makes sure that localHasNoDirectReference can actually fail if
   # it doesn't do anything. If this test fails, either the test setup was broken,
   # or Haskell packaging has changed the way `src` is treated in such a way that
@@ -39,7 +27,7 @@ lib.recurseIntoAttrs rec {
         drvPath = builtins.unsafeDiscardOutputDependency localRaw.drvPath;
       }
       ''
-        grep ${src} $drvPath >/dev/null
+        grep ${localRaw.src} $drvPath >/dev/null
         touch $out
       '';
 
@@ -49,7 +37,7 @@ lib.recurseIntoAttrs rec {
         drvPath = builtins.unsafeDiscardOutputDependency localFromCabalSdist.drvPath;
       }
       ''
-        grep -v ${src} $drvPath >/dev/null
+        grep -v ${localRaw.src} $drvPath >/dev/null
         touch $out
       '';
 }

pkgs/test/haskell/cabalSdist/generated.nix

@@ -1,4 +1,4 @@
-# nix run ../../../../..#cabal2nix -- ./.
+# nix run ../../../../..#cabal2nix -- ./local
 {
   mkDerivation,
   base,
@@ -7,10 +7,11 @@
 mkDerivation {
   pname = "local";
   version = "0.1.0.0";
-  src = ./.; # also referred to as ./local in the test; these are the same path constants
+  src = ./local;
   isLibrary = false;
   isExecutable = true;
   executableHaskellDepends = [ base ];
   description = "Nixpkgs test case";
   license = lib.licenses.mit;
+  mainProgram = "local";
 }

pkgs/test/haskell/setBuildTarget/default.nix

@@ -1,7 +1,7 @@
 { pkgs, haskellPackages }:
 
 let
-  # This can be regenerated by running `cabal2nix .` in the current directory.
+  # This can be regenerated by running `cabal2nix ./src` in the current directory.
   pkgDef =
     {
       mkDerivation,
@@ -11,19 +11,11 @@ let
     mkDerivation {
       pname = "haskell-setBuildTarget";
       version = "0.1.0.0";
-      src = lib.fileset.toSource {
-        root = ./.;
-        fileset = lib.fileset.unions [
-          ./haskell-setBuildTarget.cabal
-          ./Bar.hs
-          ./Foo.hs
-          ./Setup.hs
-        ];
-      };
+      src = ./src;
       isLibrary = false;
       isExecutable = true;
       executableHaskellDepends = [ base ];
-      license = lib.licenses.bsd3;
+      license = "unknown";
     };
 
   drv = haskellPackages.callPackage pkgDef { };

pkgs/test/make-binary-wrapper/default.nix

@@ -20,16 +20,7 @@ let
     runCommand "make-binary-wrapper-test-${testname}" env ''
       mkdir -p tmp/foo # for the chdir test
 
-      source=${
-        lib.fileset.toSource {
-          root = ./.;
-          fileset = lib.fileset.unions [
-            (./. + "/${testname}.cmdline")
-            (./. + "/${testname}.c")
-            (lib.fileset.maybeMissing (./. + "/${testname}.env"))
-          ];
-        }
-      }
+      source=${./${testname}}
 
       params=$(<"$source/${testname}.cmdline")
       eval "makeCWrapper /send/me/flags $params" > wrapper.c

pkgs/top-level/all-packages.nix

@@ -5667,7 +5667,7 @@ with pkgs;
 
   yarn-berry = callPackage ../development/tools/yarn-berry { };
 
-  yarn2nix-moretea = callPackage ../development/tools/yarn2nix-moretea/yarn2nix { pkgs = pkgs.__splicedPackages; };
+  yarn2nix-moretea = callPackage ../development/tools/yarn2nix-moretea { pkgs = pkgs.__splicedPackages; };
 
   inherit (yarn2nix-moretea)
     yarn2nix

pkgs/top-level/default.nix

@@ -27,6 +27,10 @@
 , # Allow a configuration attribute set to be passed in as an argument.
   config ? {}
 
+, # Temporary hack to let Nixpkgs forbid internal use of `lib.fileset`
+  # until <https://github.com/NixOS/nix/issues/11503> is fixed.
+  __allowFileset ? true
+
 , # List of overlays layers used to extend Nixpkgs.
   overlays ? []
 
@@ -47,7 +51,24 @@ let # Rename the function arguments
   crossSystem0 = crossSystem;
 
 in let
-  lib = import ../../lib;
+  pristineLib = import ../../lib;
+
+  lib =
+    if __allowFileset then
+      pristineLib
+    else
+      pristineLib.extend (_: _: {
+        fileset = abort ''
+
+          The use of `lib.fileset` is currently forbidden in Nixpkgs due to the
+          upstream Nix bug <https://github.com/NixOS/nix/issues/11503>. This
+          causes difficult‐to‐debug errors when combined with chroot stores,
+          such as in the NixOS installer.
+
+          For packages that require source to be vendored inside Nixpkgs,
+          please use a subdirectory of the package instead.
+        '';
+      });
 
   inherit (lib) throwIfNot;
 

pkgs/top-level/release-cross.nix

@@ -17,7 +17,10 @@
 , # Strip most of attributes when evaluating to spare memory usage
   scrubJobs ? true
 , # Attributes passed to nixpkgs. Don't build packages marked as unfree.
-  nixpkgsArgs ? { config = { allowUnfree = false; inHydra = true; }; }
+  nixpkgsArgs ? {
+    config = { allowUnfree = false; inHydra = true; };
+    __allowFileset = false;
+  }
 }:
 
 let

pkgs/top-level/release-cuda.nix

@@ -43,6 +43,8 @@ in
       "${variant}Support" = true;
       inHydra = true;
     };
+
+    __allowFileset = false;
   },
   ...
 }@args:

pkgs/top-level/release-lib.nix

@@ -3,7 +3,10 @@
 , packageSet ? (import ../..)
 , scrubJobs ? true
 , # Attributes passed to nixpkgs. Don't build packages marked as unfree.
-  nixpkgsArgs ? { config = { allowUnfree = false; inHydra = true; }; }
+  nixpkgsArgs ? {
+    config = { allowUnfree = false; inHydra = true; };
+    __allowFileset = false;
+  }
 }:
 
 let

pkgs/top-level/release-outpaths.nix

@@ -57,6 +57,8 @@ let
 
             inHydra = true;
           };
+
+          __allowFileset = false;
         };
       };
   recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };

pkgs/top-level/release-python.nix

@@ -15,6 +15,8 @@
       allowUnfree = false;
       inHydra = true;
     };
+
+    __allowFileset = false;
   },
 }:
 

pkgs/top-level/release-small.nix

@@ -19,6 +19,8 @@
       allowUnfree = false;
       inHydra = true;
     };
+
+    __allowFileset = false;
   },
 }:
 

pkgs/top-level/release-unfree-redistributable.nix

@@ -37,6 +37,8 @@
       cudaSupport = true;
       inHydra = true;
     };
+
+    __allowFileset = false;
   },
   # We only build the full package set on infrequently releasing channels.
   full ? false,

pkgs/top-level/release.nix

@@ -28,16 +28,20 @@
   # Strip most of attributes when evaluating to spare memory usage
 , scrubJobs ? true
   # Attributes passed to nixpkgs. Don't build packages marked as unfree.
-, nixpkgsArgs ? { config = {
-    allowUnfree = false;
-    inHydra = true;
-    # Exceptional unsafe packages that we still build and distribute,
-    # so users choosing to allow don't have to rebuild them every time.
-    permittedInsecurePackages = [
-      "olm-3.2.16" # see PR #347899
-      "kanidm_1_3-1.3.3"
-    ];
-  }; }
+, nixpkgsArgs ? {
+    config = {
+      allowUnfree = false;
+      inHydra = true;
+      # Exceptional unsafe packages that we still build and distribute,
+      # so users choosing to allow don't have to rebuild them every time.
+      permittedInsecurePackages = [
+        "olm-3.2.16" # see PR #347899
+        "kanidm_1_3-1.3.3"
+      ];
+    };
+
+    __allowFileset = false;
+  }
 
   # This flag, if set to true, will inhibit the use of `mapTestOn`
   # and `release-lib.packagePlatforms`.  Generally, it causes the