Hey everyone. Felt bad about opening a github issue for this, so I thought I'd bring it here...

I'm trying to get a semi-complex system set up. I am trying to get an "impermanence" set up working with ZFS as my root filesystem. I've got a 512GB NVME drive for the OS and a 4TB drive for (NAS) bulk storage. I'm setting up a mirrored vdev for the OS (400GB partition on the OS drive and 400GB partition on the storage drive). The rest of the storage drive is just a normal vdev, and I have a pool for each vdev. I was actually able to boot into my system yesterday, and things seemed to be working as expected! However, I forgot encryption. When I went back to add LUKS to the storage pool's partition, I was able to set the passphrase and unlock it, but ultimately was unable to mount the dataset under the encrypted drive. I see Starting Import ZFS pool "flashpool" and a start job hangs for a while but eventually fails and I drop into emergency mode and get Cannot open access to console, the root account is locked.

I think this may have something to do with the timing of when it's trying to import the pool, but I'm not sure; if anyone has experience with LUKS+ZFS on disko, let me know!

Here's my disko.nix:

let
  root_fs_partition = {
    size = "400G"; # GiB
    content = {
      type = "zfs";
      pool = "rootpool";
    };
  };
  boot_partition =
    { mountpoint }:
    {
      size = "1G";
      type = "EF00";
      content = {
        inherit mountpoint;
        type = "filesystem";
        format = "vfat";
        mountOptions = [ "umask=0077" ];
      };
    };
  swap_partition = {
    size = "64G";
    content = {
      type = "swap";
      resumeDevice = false;
    };
  };
  zfs_rootfs_options = {
    # Enables access control lists
    acltype = "posixacl";
    # Disables tracking the time a file is accessed (viewed/ls'ed)
    atime = "off";
    # Supposedly a bit faster than zstd at the cost of slightly less
    # compression
    compression = "lz4";
    # We'll mount datasets rather than the pool itself
    mountpoint = "none";
    # Sets extended attributes in inode instead of with hidden sidecar
    # folders
    xattr = "sa";
  };
  zfs_options = {
    # 12 is a good default value... this pertains to the physical sector
    # size of the storage device in use. It's hard to find information about
    # this for the SSD I'm using, and a test I found showed that tweaking
    # this on an SSD didn't provide any performance boost, so I'm leaving it
    # at 12.
    ashift = "12";
  };
in
{
  disko.devices = {
    disk = {
      homelab = {
        type = "disk";
        device = "/dev/disk/by-id/nvme-WD_BLACK_SN770_500GB_22127H802862";
        content = {
          type = "gpt";
          partitions = {
            ESP = boot_partition { mountpoint = "/boot"; };
            swap = swap_partition;
            zfs = root_fs_partition;
          };
        };
      };
      flash0 = {
        type = "disk";
        device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2411E89FE7EE";
        content = {
          type = "gpt";
          partitions = {
            ESP = boot_partition { mountpoint = "/boot2"; };
            zfs-mirror = root_fs_partition;
            flash = {
              size = "100%";
              content = {
                type = "luks";
                name = "flashluks";
                passwordFile = "/tmp/secret.phrase";
                settings = {
                  # Enables TRIM; does have some security concerns, but they seem minor to me
                  allowDiscards = true;
                  keyFile = "/tmp/secret.key";
                  keyFileOffset = 618;
                  keyFileSize = 2022;
                  keyFileTimeout = 30;
                  # fallbackToPassword cannot be used when boot.initrd.systemd
                  # is in use since it is implied by that option
                };
                content = {
                  type = "zfs";
                  pool = "flashpool";
                };
              };
            };
          };
        };
      };
      # spin0 = {
      #   type = "disk";
      #   device = "FIXME";
      #   content = {
      #     type = "gpt";
      #     partitions = {
      #       zfs = {
      #         size = "100%";
      #         content = {
      #           type = "zfs";
      #           pool = "spinpool";
      #         };
      #       };
      #     };
      #   };
      # };
    };
    # https://wiki.archlinux.org/title/Install_Arch_Linux_on_ZFS
    # https://jrs-s.net/2018/08/17/zfs-tuning-cheat-sheet/
    zpool = {
      rootpool = {
        type = "zpool";
        mode = {
          topology = {
            type = "topology";
            vdev = [
              {
                mode = "mirror";
                # This needs to be an absolute path value most likely
                # https://github.com/nix-community/disko/blob/380847d94ff0fedee8b50ee4baddb162c06678df/lib/types/zpool.nix#L140
                members = [
                  "/dev/disk/by-partlabel/disk-homelab-zfs"
                  "/dev/disk/by-partlabel/disk-flash0-zfs-mirror"
                ];
              }
            ];
          };
        };
        # -O options for zpool create
        rootFsOptions = zfs_rootfs_options;
        # -o options for zpool create
        options = zfs_options;
        datasets = {
          # All datasets under drop are erased on reboot
          "drop" = {
            type = "zfs_fs";
            options.mountpoint = "none";
          };
          "drop/root" = {
            type = "zfs_fs";
            mountpoint = "/";
            options."com.sun:auto-snapshot" = "false";
            postCreateHook = "zfs list -t snapshot -H -o name | grep -E '^rootpool/drop/root@blank$' || zfs snapshot rootpool/drop/root@blank";
          };
          "drop/nix" = {
            type = "zfs_fs";
            mountpoint = "/nix";
            options."com.sun:auto-snapshot" = "false";
          };
          # All datasets under keep are persisted on reboot
          "keep" = {
            type = "zfs_fs";
            options.mountpoint = "none";
          };
          "keep/keep" = {
            type = "zfs_fs";
            mountpoint = "/keep";
            options."com.sun:auto-snapshot" = "true";
          };
          "keep/home" = {
            type = "zfs_fs";
            mountpoint = "/home";
            # Used by services.zfs.autoSnapshot options.
            options."com.sun:auto-snapshot" = "true";
          };
        };
      };
      flashpool = {
        type = "zpool";
        # -O options for zpool create
        rootFsOptions = zfs_rootfs_options;
        # -o options for zpool create
        options = zfs_options;
        datasets = {
          "flashroot" = {
            type = "zfs_fs";
            options.mountpoint = "none";
          };
          "flashroot/flash" = {
            type = "zfs_fs";
            mountpoint = "/flash";
            options."com.sun:auto-snapshot" = "true";
          };
        };
      };
    };
  };
}

Hey everyone. Felt bad about opening a github issue for this, so I thought I'd bring it here...

I'm trying to get a semi-complex system set up. I am trying to get an "impermanence" set up working with ZFS as my root filesystem. I've got a 512GB NVME drive for the OS and a 4TB drive for (NAS) bulk storage. I'm setting up a mirrored vdev for the OS (400GB partition on the OS drive and 400GB partition on the storage drive). The rest of the storage drive is just a normal vdev, and I have a pool for each vdev. I was actually able to boot into my system yesterday, and things seemed to be working as expected! However, I forgot encryption. When I went back to add LUKS to the storage pool's partition, I was able to set the passphrase and unlock it, but ultimately was unable to mount the dataset under the encrypted drive. I see Starting Import ZFS pool "flashpool" and a start job hangs for a while but eventually fails and I drop into emergency mode and get Cannot open access to console, the root account is locked.

I think this may have something to do with the timing of when it's trying to import the pool, but I'm not sure. I also suspect using legacy mountpoints could potentially fix it? If anyone has experience with LUKS+ZFS on disko, let me know!

/nix/store/bz18paxklav6yyhqvvl1l7aiq3drncy4-disko-install/bin/.disko-install-wrapped: line 216: artifacts[1]: unbound variable

[nixos@nixos:~/infra]$ sed -n '214, 220 p' /nix/store/bz18paxklav6yyhqvvl1l7aiq3drncy4-disko-install/bin/.disko-install-wrapped
  IFS=$'\n' mapfile -t artifacts <<<"$outputs"
  nixos_system=${artifacts[0]}
  closure_info=${artifacts[1]}
  disko_script=${artifacts[2]}

  if [[ -n ${dry_run-} ]]; then
    echo "Would run: $disko_script"

I have a disks.nix in my git repo, added it to my configuration, and disko-install ... worked perfectly! My my disks config looks like (basically it is the luks-btrfs example in the docs):

{
  disko.devices = {
    disk = {
      main = {
        type = "disk";
        # When using disko-install, we can overwrite this value (device) from the commandline:
        # $ sudo nix run 'github:nix-community/disko/latest#disko-install' -- --write-efi-boot-entries --flake '/tmp/config/etc/nixos#mymachine' --disk main /dev/sda
        # see: https://github.com/nix-community/disko/blob/master/docs/disko-install.md
        device = "/dev/nvme0n1";
        content = {
          type = "gpt";
          partitions = {
            ESP = {
              size = "512M";
              type = "EF00";
              content = {
                type = "filesystem";
                format = "vfat";
                mountpoint = "/boot";
                mountOptions = [ "umask=0077" ];
              };
            };
            luks = {
              size = "100%";
              content = {
                type = "luks";
                name = "crypted";
                # disable settings.keyFile if you want to use interactive password entry
                #passwordFile = "/tmp/secret.key"; # Interactive
                settings = {
                  allowDiscards = true;
                  # keyFile = "/tmp/secret.key";
                };
                # additionalKeyFiles = [ "/tmp/additionalSecret.key" ];
                content = {
                  type = "btrfs";
                  extraArgs = [ "-f" ];
                  subvolumes = {
                    "/root" = {
                      mountpoint = "/";
                      mountOptions = [ "compress=zstd" "noatime" ];
                    };
                    "/home" = {
                      mountpoint = "/home";
                      mountOptions = [ "compress=zstd" "noatime" ];
                    };
                    "/nix" = {
                      mountpoint = "/nix";
                      mountOptions = [ "compress=zstd" "noatime" ];
                    };
                    "/swap" = {
                      mountpoint = "/.swapvol";
                      swap.swapfile.size = "8G";
                    };
                  };
                };
              };
            };
          };
        };
      };
    };
  };
}