fex: fix tests + minimize set of cloned submodules (#374694)

pkgs/by-name/fe/fex/package.nix

@@ -8,6 +8,9 @@
   qt5,
   python3,
   nix-update-script,
+  xxHash,
+  fmt,
+  nasm,
 }:
 
 llvmPackages.stdenv.mkDerivation (finalAttrs: {
@@ -18,8 +21,32 @@ llvmPackages.stdenv.mkDerivation (finalAttrs: {
     owner = "FEX-Emu";
     repo = "FEX";
     tag = "FEX-${finalAttrs.version}";
-    hash = "sha256-tqUJBHYSRlEUaLI4WItzotIHGMUNbdjA7o9NjBYZmHw=";
-    fetchSubmodules = true;
+
+    hash = "sha256-oXducy4uvf/3Ox6AadPWNl9450D9TiPIr53P91/qEvw=";
+
+    leaveDotGit = true;
+    postFetch = ''
+      cd $out
+      git reset
+
+      # Only fetch required submodules
+      git submodule update --init --depth 1 \
+        External/Vulkan-Headers \
+        External/drm-headers \
+        External/jemalloc \
+        External/jemalloc_glibc \
+        External/robin-map \
+        External/vixl \
+        Source/Common/cpp-optparse \
+        External/Catch2
+
+      find . -name .git -print0 | xargs -0 rm -rf
+
+      # Remove some more unnecessary directories
+      rm -r \
+        External/vixl/src/aarch32 \
+        External/vixl/test
+    '';
   };
 
   nativeBuildInputs = [
@@ -37,12 +64,19 @@ llvmPackages.stdenv.mkDerivation (finalAttrs: {
     ))
   ];
 
-  buildInputs = with qt5; [
-    qtbase
-    qtdeclarative
-    qtquickcontrols
-    qtquickcontrols2
-  ];
+  nativeCheckInputs = [ nasm ];
+
+  buildInputs =
+    [
+      xxHash
+      fmt
+    ]
+    ++ (with qt5; [
+      qtbase
+      qtdeclarative
+      qtquickcontrols
+      qtquickcontrols2
+    ]);
 
   cmakeFlags = [
     (lib.cmakeFeature "CMAKE_BUILD_TYPE" "Release")
@@ -54,7 +88,18 @@ llvmPackages.stdenv.mkDerivation (finalAttrs: {
   ];
 
   strictDeps = true;
-  doCheck = false; # broken on Apple silicon computers
+
+  # Unsupported on non-4K page size kernels (e.g. Apple Silicon)
+  doCheck = true;
+
+  # List not exhaustive, e.g. because they depend on an x86 compiler or some
+  # other difficult-to-build test binaries.
+  checkTarget = lib.concatStringsSep " " [
+    "asm_tests"
+    "api_tests"
+    "fexcore_apitests"
+    "emitter_tests"
+  ];
 
   # Avoid wrapping anything other than FEXConfig, since the wrapped executables
   # don't seem to work when registered as binfmts.