In reply to @ibizaman:matrix.org
chinchilla optional I would put all that in your ssh config file. This usually always works vs. environment variables. Can you post your exact ssh command? 

❯ colmena build -v --show-trace
[INFO ] Using configuration: /home/hexa/git/chaos/infra-nix/hive.nix
[INFO ] Enumerating nodes...
error:
       … while evaluating attribute 'archive.darmstadt.ccc.de'

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/attrsets.nix:560:14:

          559|     value:
          560|     { inherit name value; };
             |              ^
          561|

       … while calling anonymous lambda

         at /run/user/1000/colmena-assets-LXWf6W/eval.nix:188:39:

          187|   toplevel =         lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;
          188|   deploymentConfig = lib.mapAttrs (_: v: v.config.deployment)            nodes;
             |                                       ^
          189|   deploymentConfigSelected = names: lib.filterAttrs (name: _: elem name names) deploymentConfig;

       … while calling 'g'

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/attrsets.nix:701:19:

          700|           g =
          701|             name: value:
             |                   ^
          702|             if isAttrs value && cond value

       … from call site

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/attrsets.nix:704:20:

          703|               then recurse (path ++ [name]) value
          704|               else f (path ++ [name]) value;
             |                    ^
          705|         in mapAttrs g;

       … while calling anonymous lambda

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/modules.nix:242:72:

          241|           # For definitions that have an associated option
          242|           declaredConfig = mapAttrsRecursiveCond (v: ! isOption v) (_: v: v.value) options;
             |                                                                        ^
          243|

       … while evaluating the option `_module.freeformType':

       … while calling anonymous lambda

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/modules.nix:538:19:

          537|     mergeModules' prefix modules
          538|       (concatMap (m: map (config: { file = m._file; inherit config; }) (pushDownProperties m.config)) modules);
             |                   ^
          539|

       … from call site

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/modules.nix:538:73:

          537|     mergeModules' prefix modules
          538|       (concatMap (m: map (config: { file = m._file; inherit config; }) (pushDownProperties m.config)) modules);
             |                                                                         ^
          539|

       … while calling 'pushDownProperties'

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/modules.nix:874:24:

          873|   */
          874|   pushDownProperties = cfg:
             |                        ^
          875|     if cfg._type or "" == "merge" then

       … from call site

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/nixos/lib/eval-config.nix:50:8:

           49|     config = lib.mkMerge (
           50|       (optional (system != null) {
             |        ^
           51|         # Explicit `nixpkgs.system` or `nixpkgs.localSystem` should override

       … while calling 'optional'

         at /nix/store/fgdyf58bmkl0y2ykl06fmzqjrjv9kqf3-nixos-23.11-src/lib/lists.nix:359:20:

          358|   */
          359|   optional = cond: elem: if cond then [elem] else [];
             |                    ^
          360|

       error: attribute 'system' missing

       at /run/user/1000/colmena-assets-LXWf6W/eval.nix:143:18:

          142|     };
          143|   in evalConfig {
             |                  ^
          144|     inherit (npkgs) system;
       Did you mean one of mystem, systemc, systemd, syntex or systemfd?
[ERROR] -----
[ERROR] Operation failed with error: Child process exited with error code: 1
Hint: Backtrace available - Use `RUST_BACKTRACE=1` environment variable to display a backtrace

let
  sources = import ./nix/sources.nix;

  defaultSystem = "x86_64-linux";
  defaultPkgs = sources."nixos-23.11";

  lib = import (defaultPkgs + "/lib");

  machines = {
    "archive.darmstadt.ccc.de" = {};
    "bitwarden.darmstadt.ccc.de" = {};
    "md.darmstadt.ccc.de" = {};
    "monitoring.darmstadt.ccc.de" = {};
    "mx1.darmstadt.ccc.de" = {
      system = "aarch64-linux";
    };
    "proxmox.darmstadt.ccc.de" = {};
    "sso1.darmstadt.ccc.de" = {};
    "sso2.darmstadt.ccc.de" = {};
    "todo.darmstadt.ccc.de" = {};
    "wiki.darmstadt.ccc.de" = {};
  };

  # per machine entrypoint
  mkMachine = hostName: {
    system ? defaultSystem,
    packages ? defaultPkgs,
    ...
  }: {
    imports = [
      ./common.nix
      ./machines/${hostName}
    ];

    nixpkgs = {
      inherit system;
    };
  };

in

{
  meta = {
    # https://colmena.cli.rs/unstable/tutorial/index.html#basic-configuration
    nixpkgs = import defaultPkgs {};

    # Allow overriding package set and system architecture per machine
    nodeNixpkgs = lib.mapAttrs (hostName: {
      packages ? defaultPkgs,
      system ? defaultSystem,
      ...
    }: (import packages { inherit system; }))
    machines;
  };

}
// lib.mapAttrs mkMachine machines

let
  sources = import ./nix/sources.nix;

  defaultSystem = "x86_64-linux";
  defaultPkgs = sources."nixos-23.11";

  lib = import (defaultPkgs + "/lib");

  machines = {
    "archive.darmstadt.ccc.de" = {};
    "bitwarden.darmstadt.ccc.de" = {};
    "md.darmstadt.ccc.de" = {};
    "monitoring.darmstadt.ccc.de" = {};
    "mx1.darmstadt.ccc.de" = {
      system = "aarch64-linux";
    };
    "proxmox.darmstadt.ccc.de" = {};
    "sso1.darmstadt.ccc.de" = {};
    "sso2.darmstadt.ccc.de" = {};
    "todo.darmstadt.ccc.de" = {};
    "wiki.darmstadt.ccc.de" = {};
  };

  # per machine entrypoint
  mkMachine = hostName: {
    system ? defaultSystem,
    packages ? defaultPkgs,
    ...
  }: {
    imports = [
      ./common.nix
      ./machines/${hostName}
    ];

    nixpkgs = {
      inherit system;
    };
  };

in

{
  meta = {
    # https://colmena.cli.rs/unstable/tutorial/index.html#basic-configuration
    nixpkgs = import defaultPkgs {};

    # Allow overriding package set and system architecture per machine
    nodeNixpkgs = lib.mapAttrs (hostName: {
      packages ? defaultPkgs,
      system ? defaultSystem,
      ...
    }: (import packages { inherit system; }))
    machines;
  };

}
// lib.mapAttrs mkMachine machines

  error: assertion '(((args) ? localSystem) -> (! ((args) ? system)))' failed