Here is the updated working example in case it helps anyone in the future:

{
  pkgs ? import <nixpkgs> { },
  lib ? pkgs.lib,
}:

let
  testSubmodule =
    { config, ... }:
    {
      freeformType = lib.types.lazyAttrsOf lib.types.raw;

      options.a = lib.mkOption {
        type = lib.types.int;
        default = 0;
      };
    };

  isSubmoduleFn =
    m:
    let
      args = builtins.functionArgs m;
    in
    args ? lib || args ? config || args ? options;

  customSubmoduleWith =
    attrs:
    let
      t = lib.types.addCheck (lib.types.submoduleWith attrs) (v: builtins.isAttrs v || isSubmoduleFn v);
    in
    t
    // rec {
      substSubmodules = m: customSubmoduleWith (attrs // { modules = m; });
      functor = t.functor // {
        type = customSubmoduleWith;
      };
      typeMerge = lib.types.defaultTypeMerge functor;
    };

  customSubmodule =
    module:
    customSubmoduleWith {
      shorthandOnlyDefinesConfig = true;
      modules = [ module ];
    };

  staticInnerType = customSubmodule testSubmodule;

  dynamicInnerType = lib.types.functionTo staticInnerType;

  innerType = lib.types.either staticInnerType dynamicInnerType;

  testType = lib.types.lazyAttrsOf innerType;

  optionModule1 = {
    options.test = lib.mkOption {
      type = testType;
    };
  };

  optionModule2 = {
    options.test = lib.mkOption {
      type =
        let
          t = customSubmodule {
            options.b = lib.mkOption {
              type = lib.types.bool;
            };
          };
        in
        lib.types.lazyAttrsOf (lib.types.either t (lib.types.functionTo t));
    };
  };

  staticConfig = {
    config.test.random2 = {
      b = false;
    };
  };

  dynamicConfig = {
    config.test.random1 =
      { arg1 }:
      {
        a = 1;
      };
  };

  result = lib.evalModules {
    modules = [
      optionModule1
      optionModule2

      staticConfig
      dynamicConfig
    ];
  };
in
result

using some fixpoint mechanism like overlays?

Maybe mkOptionType could apply mkExtensible, which adds the .extend attribute? addCheck could then apply an overlay to <type>.extend 🤔

E.g.

{
  options.foo = mkOption {
    type = submodule {
      # This module is in `getSubOptions` _and_ `valueMeta.configuration`
    };
  };
  config.foo = {
    # This module is only in `valueMeta.configuration`, not `getSubOptions`
  };
}

I feel like there's a subtle confusion going on here with what we mean by "definition".

A submodule is just another configuration, within a wider configuration, evaluated mostly in isolation.

Both the outer and sub configurations have modules, and those modules can contain config definitions for their respective configurations.

When I say valueMeta.configuration considers modules from config definitions, I mean config definitions in the outer configuration; definitions that define modules for the submodule to evaluate.

I feel like there's a subtle confusion going on here with what we mean by "definition".

A submodule is just another configuration, within a wider configuration, evaluated mostly in isolation.

The submodule type merges to its configuration's config value, and the configuration as a whole is exposed through valueMeta, which comes from v2 check+merge.

Both the outer and sub configurations have modules, and those modules can contain config definitions for their respective configurations.

When I say valueMeta considers modules from config definitions, I mean config definitions in the outer configuration; definitions that define modules for the submodule to evaluate.

To oversimplify, a submodule-type's lifecycle is:

1. the type is instantiated, with initial modules
   • options declared and defaults defined are visible to type.getSubOptions
2. an option is declared (in a host configuration) using the type
3. the option is defined (either via config or default, in the host configuration)
   • these definitions are modules
4. the option is merged, using the type's merge function
   1. it extends its base configuration with the defined modules
   2. it exposes the final configuration via valueMeta = { inherit configuration; }
   3. it evaluates the final value as configuration.config