FYI: /usr/sbin/sshd -ddd -p 2222 was quite helpful, turned out SSHD didn't like the /nix/store permissions, I changed it to this copying and referencing static path:

          system.activationScripts.preActivation.text = ''
            cp -a ${pkgs.writeShellScript "cat-nofail" ''/bin/cat "$@" || :''} /etc/ssh/authorized-keys-command
          '';

FYI: I found it:

            virtualisation.qemu.options = [
              # socat - UNIX-CONNECT:/run/org.nixos.linux-builder/qemu-serial.sock
              # minicom -D 'unix#/run/org.nixos.linux-builder/qemu-serial.sock'
              ''-serial unix:"$TMPDIR/qemu-serial.sock",server,nowait''
            ];

    [1/2] /nix/store/jk2r3w2q06vh7hkfxrw74ckrlrppm6gz-swiftc-6.2.3/bin/swiftc -j 16 -num-threads 16 -c  -module-name cmTC_a937f -target arm64-apple-macosx14.0  -sdk /nix/store/i6yfk1parrl2f2m>
    FAILED: [code=1] CMakeFiles/cmTC_a937f.dir/main.swift.o
    /nix/store/jk2r3w2q06vh7hkfxrw74ckrlrppm6gz-swiftc-6.2.3/bin/swiftc -j 16 -num-threads 16 -c  -module-name cmTC_a937f -target arm64-apple-macosx14.0  -sdk /nix/store/i6yfk1parrl2f2mhj96x5>
    <unknown>:0: warning: using (deprecated) legacy driver, Swift installation does not contain swift-driver at: '/nix/store/jk2r3w2q06vh7hkfxrw74ckrlrppm6gz-swiftc-6.2.3/bin/swift-driver-new'
    <unknown>:0: warning: option '-incremental' is only supported in swift-driver
    /nix/store/i6yfk1parrl2f2mhj96x565ijc3lg7xv-apple-sdk-26.0/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib/swift/Swift.swiftmodule/arm64e-apple-macos.swiftinterface:5:8: error>
        3 | // swift-module-flags: -target arm64e-apple-macosx26.0 -target-variant arm64e-apple-ios26.0-macabi -enable-objc-interop -enable-library-evolution -module-link-name swiftCore -pars>
        4 | // swift-module-flags-ignorable: -enable-lexical-lifetimes=false -enable-ossa-modules -strict-memory-safety -formal-cxx-interoperability-mode=off -interface-compiler-version 6.2
        5 | import SwiftShims
          |        `- error: no such module 'SwiftShims'
        6 | @inlinable public func min<T>(_ x: T, _ y: T) -> T where T : Swift.Comparable {
        7 |

    /nix/store/i6yfk1parrl2f2mhj96x565ijc3lg7xv-apple-sdk-26.0/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib/swift/Swift.swiftmodule/arm64e-apple-macos.swiftinterface:1:1: error>
        1 | // swift-interface-format-version: 1.0
          | `- error: failed to build module 'Swift'; this SDK is not supported by the compiler (the SDK is built with 'Apple Swift version 6.2 effective-5.10 (swiftlang-6.2.0.17.14 clang-170>
        2 | // swift-compiler-version: Apple Swift version 6.2 effective-5.10 (swiftlang-6.2.0.17.14 clang-1700.3.17.1)
        3 | // swift-module-flags: -target arm64e-apple-macosx26.0 -target-variant arm64e-apple-ios26.0-macabi -enable-objc-interop -enable-library-evolution -module-link-name swiftCore -pars>
    ninja: build stopped: subcommand failed.

huh what gives?

$ uname -a
Darwin MacBookPro.Home 24.6.0 Darwin Kernel Version 24.6.0: Mon Jul 14 11:30:29 PDT 2025; root:xnu-11417.140.69~1/RELEASE_ARM64_T6000 arm64
$ nix-shell -p hello --run 'uname -a'
Darwin MacBookPro.Home 24.6.0 Darwin Kernel Version 24.6.0: Mon Jul 14 11:30:29 PDT 2025; root:xnu-11417.140.69~1/RELEASE_ARM64_T6000 arm64 arm Darwin

one part of it is having a wrapper to get the sockets

// Simple wrapper to activate launchd sockets
// and set them up in the same way systemd would
// so that we can use gpg-agent in --supervised mode

#include <errno.h>
#include <err.h>
#include <unistd.h>
#include <launch.h>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>

int get_launchd_socket(const char *sockName)
{
  // Get our sockets from launchd
  int *fds = NULL;
  size_t count = 0;
  errno = launch_activate_socket(sockName, &fds, &count);

  if (errno != 0 || fds == NULL || count < 1)
  {
    warn("Error getting socket FD from launchd");
    return 0;
  }

  if (count != 1)
  {
    warnx("Expected one FD from launchd, got %zu. Only using first socket.", count);
  }

  // Unset FD_CLOEXEC bit
  fcntl(fds[0], F_SETFD, fcntl(fds[0], F_GETFD, 0) & ~FD_CLOEXEC);

  if (fds)
  {
    free(fds);
  }

  return 1;
}

int main(int argc, char **argv)
{
  // List of sockets we're going to check for
  const char *sockets[] = {
      "ssh",
      "browser",
      "extra",
      "std"};
  int fds = 0;
  char *fdsString = NULL;
  char *fdNames = NULL;
  char *tmpfdNames = NULL;

  // Activate the sockets and count and store names
  for (int i = 0; i < sizeof(sockets) / sizeof(sockets[0]); i++)
  {
    if (get_launchd_socket(sockets[i]))
    {
      fds++;
      asprintf(&fdNames, (tmpfdNames == NULL ? "%s%s" : "%s:%s"), (tmpfdNames == NULL ? "" : tmpfdNames), sockets[i]);
      if (tmpfdNames)
      {
        free(tmpfdNames);
      }
      tmpfdNames = fdNames;
    }
  }

  // Set the ENV var for our PID
  char *pidString = NULL;
  asprintf(&pidString, "%ld", (long)getpid());
  setenv("LISTEN_PID", pidString, 0);
  free(pidString);

  // Set the number of FDs we've opened
  asprintf(&fdsString, "%d", fds);
  setenv("LISTEN_FDS", fdsString, 0);
  free(fdsString);

  // And their names
  setenv("LISTEN_FDNAMES", (fdNames == NULL ? "" : fdNames), 0);
  free(fdNames);

  // Launch the command we were passed
  ++argv;
  if (*argv)
  {
    execvp(*argv, argv);
    err(1, "Error executing command");
  }
  else
  {
    errx(1, "No command specified");
  }
}