Sadness.

I had to package the 12.8 version of cuda-bindings, because we're on cuda 12.8.
However, it dlopens libcuda.so and the driver (on my system at least) tells it that it uses cuda 13.1.
Hence, it errors out :/

https://github.com/NixOS/nixpkgs/pull/482638

(cuda-bindings is supposed to be required in the new torch update, not sure how critical it is though)

Here's the stack trace when I re-enable the failing test:

______________________ ERROR collecting test_nvjitlink.py ______________________
tests/test_nvjitlink.py:62: in <module>
    not check_nvjitlink_usable(), reason="nvJitLink not usable, maybe not installed or too old (<12.3)"
        ^^^^^^^^^^^^^^^^^^^^^^^^
tests/test_nvjitlink.py:58: in check_nvjitlink_usable
    return inner_nvjitlink._inspect_function_pointer("__nvJitLinkVersion") != 0
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
cuda/bindings/_internal/nvjitlink.pyx:254: in cuda.bindings._internal.nvjitlink._inspect_function_pointer
    ???
cuda/bindings/_internal/nvjitlink.pyx:257: in cuda.bindings._internal.nvjitlink._inspect_function_pointer
    ???
cuda/bindings/_internal/nvjitlink.pyx:205: in cuda.bindings._internal.nvjitlink._inspect_function_pointers
    ???
cuda/bindings/_internal/nvjitlink.pyx:99: in cuda.bindings._internal.nvjitlink._check_or_init_nvjitlink
    ???
cuda/bindings/_internal/nvjitlink.pyx:57: in cuda.bindings._internal.nvjitlink.load_library
    ???
cuda/bindings/_internal/utils.pyx:137: in cuda.bindings._internal.utils.get_nvjitlink_dso_version_suffix
    ???
E   cuda.bindings._internal.utils.NotSupportedError: only CUDA 12 driver is supported
------------------------------- Captured stdout --------------------------------
+++++++++++++++ 13010

hm i'm getting an infinite hang on pytestCheckPhase for python3.12-keras3.13.1 since the 3.13.1 update 🤔 https://github.com/NixOS/nixpkgs/commit/73806a397d9f9360f331ed98202e5cecc1230548
building python3.12-keras-3.13.1 (pytestCheckPhase): keras/src/trainers/trainer_test.py::TestTrainer::test_fit_with_data_adapter_py_dataset_infinite_cw PASSED [ 94%]
hang happens every time

nix-top shows

  UID     PID    PPID STIME     TIME COMMAND
30001 3422767 3422639 15:44 00:00:00 bash -e /nix/store/l622p70vy8k5sh7y5wizi5f2mic6ynpg-source-stdenv.sh /nix/store/shkw4qm9qcw5sc5n1k5jznc83ny02r39-default-builder.sh
30001 3425989 3422767 15:45 00:17:09 /nix/store/iizfzmqjdfvbyx82gqw92glb5zl6m4my-python3-3.12.12/bin/python3.12 -m pytest --ignore-glob=keras/src/layers/preprocessing/data_layer_test.py --ignore-glob=keras/src/layers/preprocessing/image_preprocessing/resizing_test.py --ignore-glob=keras/src/layers/preprocessing/rescaling_test.py --ignore-glob=keras/src/trainers/data_adapters/grain_dataset_adapter_test.py --ignore-glob=keras/src/export/onnx_test.py --ignore-glob=integration_tests/dataset_tests --ignore-glob=keras/src/applications/applications_test.py --ignore-glob=integration_tests/jax_custom_fit_test.py --ignore-glob=integration_tests/tf_distribute_training_test.py --ignore-glob=integration_tests/torch_custom_fit_test.py --ignore-glob=integration_tests/torch_workflow_test.py --ignore-glob=keras/src/backend/tensorflow/saved_model_test.py -k not (test_fit_with_data_adapter_grain_dataloader) and not (test_fit_with_data_adapter_grain_datast) and not (test_fit_with_data_adapter_grain_datast_with_len) and not (test_image_dataset_from_directory_binary_grain) and not (test_image_dataset_from_directory_color_modes_grain) and not (test_image_dataset_from_directory_crop_to_aspect_ratio_grain) and not (test_image_dataset_from_directory_follow_links_grain) and not (test_image_dataset_from_directory_manual_labels_grain) and not (test_image_dataset_from_directory_multiclass_grain) and not (test_image_dataset_from_directory_no_labels_grain) and not (test_image_dataset_from_directory_not_batched_grain) and not (test_image_dataset_from_directory_pad_to_aspect_ratio_grain) and not (test_image_dataset_from_directory_shuffle_grain) and not (test_image_dataset_from_directory_validation_split_grain) and not (test_sample_count_grain) and not (test_text_dataset_from_directory_binary_grain) and not (test_text_dataset_from_directory_follow_links_grain) and not (test_text_dataset_from_directory_manual_labels_grain) and not (test_text_dataset_from_directory_multiclass_grain) and not (test_text_dataset_from_directory_not_batched_grain) and not (test_text_dataset_from_directory_standalone_grain) and not (test_text_dataset_from_directory_validation_split_grain) and not (test_keras_imports) and not (test_reloading_default_saved_model)
30001 3797431 3425989 15:59 00:00:00 /nix/store/iizfzmqjdfvbyx82gqw92glb5zl6m4my-python3-3.12.12/bin/python3.12 -m pytest --ignore-glob=keras/src/layers/preprocessing/data_layer_test.py --ignore-glob=keras/src/layers/preprocessing/image_preprocessing/resizing_test.py --ignore-glob=keras/src/layers/preprocessing/rescaling_test.py --ignore-glob=keras/src/trainers/data_adapters/grain_dataset_adapter_test.py --ignore-glob=keras/src/export/onnx_test.py --ignore-glob=integration_tests/dataset_tests --ignore-glob=keras/src/applications/applications_test.py --ignore-glob=integration_tests/jax_custom_fit_test.py --ignore-glob=integration_tests/tf_distribute_training_test.py --ignore-glob=integration_tests/torch_custom_fit_test.py --ignore-glob=integration_tests/torch_workflow_test.py --ignore-glob=keras/src/backend/tensorflow/saved_model_test.py -k not (test_fit_with_data_adapter_grain_dataloader) and not (test_fit_with_data_adapter_grain_datast) and not (test_fit_with_data_adapter_grain_datast_with_len) and not (test_image_dataset_from_directory_binary_grain) and not (test_image_dataset_from_directory_color_modes_grain) and not (test_image_dataset_from_directory_crop_to_aspect_ratio_grain) and not (test_image_dataset_from_directory_follow_links_grain) and not (test_image_dataset_from_directory_manual_labels_grain) and not (test_image_dataset_from_directory_multiclass_grain) and not (test_image_dataset_from_directory_no_labels_grain) and not (test_image_dataset_from_directory_not_batched_grain) and not (test_image_dataset_from_directory_pad_to_aspect_ratio_grain) and not (test_image_dataset_from_directory_shuffle_grain) and not (test_image_dataset_from_directory_validation_split_grain) and not (test_sample_count_grain) and not (test_text_dataset_from_directory_binary_grain) and not (test_text_dataset_from_directory_follow_links_grain) and not (test_text_dataset_from_directory_manual_labels_grain) and not (test_text_dataset_from_directory_multiclass_grain) and not (test_text_dataset_from_directory_not_batched_grain) and not (test_text_dataset_from_directory_standalone_grain) and not (test_text_dataset_from_directory_validation_split_grain) and not (test_keras_imports) and not (test_reloading_default_saved_model)
30001 3797433 3425989 15:59 00:00:00 /nix/store/iizfzmqjdfvbyx82gqw92glb5zl6m4my-python3-3.12.12/bin/python3.12 -m pytest --ignore-glob=keras/src/layers/preprocessing/data_layer_test.py --ignore-glob=keras/src/layers/preprocessing/image_preprocessing/resizing_test.py --ignore-glob=keras/src/layers/preprocessing/rescaling_test.py --ignore-glob=keras/src/trainers/data_adapters/grain_dataset_adapter_test.py --ignore-glob=keras/src/export/onnx_test.py --ignore-glob=integration_tests/dataset_tests --ignore-glob=keras/src/applications/applications_test.py --ignore-glob=integration_tests/jax_custom_fit_test.py --ignore-glob=integration_tests/tf_distribute_training_test.py --ignore-glob=integration_tests/torch_custom_fit_test.py --ignore-glob=integration_tests/torch_workflow_test.py --ignore-glob=keras/src/backend/tensorflow/saved_model_test.py -k not (test_fit_with_data_adapter_grain_dataloader) and not (test_fit_with_data_adapter_grain_datast) and not (test_fit_with_data_adapter_grain_datast_with_len) and not (test_image_dataset_from_directory_binary_grain) and not (test_image_dataset_from_directory_color_modes_grain) and not (test_image_dataset_from_directory_crop_to_aspect_ratio_grain) and not (test_image_dataset_from_directory_follow_links_grain) and not (test_image_dataset_from_directory_manual_labels_grain) and not (test_image_dataset_from_directory_multiclass_grain) and not (test_image_dataset_from_directory_no_labels_grain) and not (test_image_dataset_from_directory_not_batched_grain) and not (test_image_dataset_from_directory_pad_to_aspect_ratio_grain) and not (test_image_dataset_from_directory_shuffle_grain) and not (test_image_dataset_from_directory_validation_split_grain) and not (test_sample_count_grain) and not (test_text_dataset_from_directory_binary_grain) and not (test_text_dataset_from_directory_follow_links_grain) and not (test_text_dataset_from_directory_manual_labels_grain) and not (test_text_dataset_from_directory_multiclass_grain) and not (test_text_dataset_from_directory_not_batched_grain) and not (test_text_dataset_from_directory_standalone_grain) and not (test_text_dataset_from_directory_validation_split_grain) and not (test_keras_imports) and not (test_reloading_default_saved_model)