From 5a8a58e55c78b841b80485c14e72867a5d55ede8 Mon Sep 17 00:00:00 2001 From: Jinfeng Date: Thu, 5 Nov 2026 01:35:37 +0000 Subject: [PATCH 1/5] backport 3308 to 12.9.x --- cuda_bindings/cuda/bindings/driver.pyx | 23 +++++++--- cuda_bindings/cuda/bindings/runtime.pyx.in | 18 ++++++-- .../docs/source/release/12.X.Y-notes.rst | 2 + cuda_bindings/tests/test_callback_threads.py | 43 +++++++++++++++++++ 4 files changed, 77 insertions(+), 9 deletions(-) create mode 100644 cuda_bindings/tests/test_callback_threads.py diff --git a/cuda_bindings/cuda/bindings/driver.pyx b/cuda_bindings/cuda/bindings/driver.pyx index 251f5c481cd..bb1fb3b27c0 100644 --- a/cuda_bindings/cuda/bindings/driver.pyx +++ b/cuda_bindings/cuda/bindings/driver.pyx @@ -28650,7 +28650,7 @@ def cuDeviceRegisterAsyncNotification(device, callbackFunc, userData): if err != cydriver.CUDA_SUCCESS: free(cbData) else: - m_global._allocated[int(callback)] = cbData + m_global._insert(int(callback), cbData) _helper_input_void_ptr_free(&cyuserDataHelper) if err != cydriver.CUDA_SUCCESS: return (_CUresult(err), None) @@ -28699,8 +28699,7 @@ def cuDeviceUnregisterAsyncNotification(device, callback): with nogil: err = cydriver.cuDeviceUnregisterAsyncNotification(cydevice, cycallback) if err == cydriver.CUDA_SUCCESS: - free(m_global._allocated[pcallback]) - m_global._allocated.erase(pcallback) + free(m_global._take(pcallback)) return (_CUresult(err),) @cython.embedsignature(True) @@ -51019,7 +51018,7 @@ def cuLogsRegisterCallback(callbackFunc, userData): if err != cydriver.CUDA_SUCCESS: free(cbData) else: - m_global._allocated[int(callback_out)] = cbData + m_global._insert(int(callback_out), cbData) _helper_input_void_ptr_free(&cyuserDataHelper) if err != cydriver.CUDA_SUCCESS: return (_CUresult(err), None) @@ -51050,8 +51049,7 @@ def cuLogsUnregisterCallback(callback): with nogil: err = cydriver.cuLogsUnregisterCallback(cycallback) if err == cydriver.CUDA_SUCCESS: - free(m_global._allocated[pcallback]) - m_global._allocated.erase(pcallback) + free(m_global._take(pcallback)) return (_CUresult(err),) @cython.embedsignature(True) @@ -52490,6 +52488,19 @@ def cuGraphicsVDPAURegisterOutputSurface(vdpSurface, unsigned int flags): cdef class cudaBindingsDriverGlobal: cdef map[void_ptr, void*] _allocated + cdef cython.pymutex _lock + + cdef void _insert(self, void_ptr key, void* value) noexcept: + with self._lock: + self._allocated[key] = value + + cdef void* _take(self, void_ptr key) noexcept: + cdef void* value = NULL + with self._lock: + if self._allocated.count(key) != 0: + value = self._allocated[key] + self._allocated.erase(key) + return value def __dealloc__(self): for item in self._allocated: diff --git a/cuda_bindings/cuda/bindings/runtime.pyx.in b/cuda_bindings/cuda/bindings/runtime.pyx.in index 284e5f91e13..aa6711d3a3b 100644 --- a/cuda_bindings/cuda/bindings/runtime.pyx.in +++ b/cuda_bindings/cuda/bindings/runtime.pyx.in @@ -20167,7 +20167,7 @@ def cudaDeviceRegisterAsyncNotification(int device, callbackFunc, userData): if err != cyruntime.cudaSuccess: free(cbData) else: - m_global._allocated[int(callback)] = cbData + m_global._insert(int(callback), cbData) _helper_input_void_ptr_free(&cyuserDataHelper) if err != cyruntime.cudaSuccess: @@ -20212,8 +20212,7 @@ def cudaDeviceUnregisterAsyncNotification(int device, callback): with nogil: err = cyruntime.cudaDeviceUnregisterAsyncNotification(device, cycallback) if err == cyruntime.cudaSuccess: - free(m_global._allocated[pcallback]) - m_global._allocated.erase(pcallback) + free(m_global._take(pcallback)) return (_cudaError_t(err),) {{endif}} @@ -39171,6 +39170,19 @@ def getLocalRuntimeVersion(): cdef class cudaBindingsRuntimeGlobal: cdef map[void_ptr, void*] _allocated + cdef cython.pymutex _lock + + cdef void _insert(self, void_ptr key, void* value) noexcept: + with self._lock: + self._allocated[key] = value + + cdef void* _take(self, void_ptr key) noexcept: + cdef void* value = NULL + with self._lock: + if self._allocated.count(key) != 0: + value = self._allocated[key] + self._allocated.erase(key) + return value def __dealloc__(self): for item in self._allocated: diff --git a/cuda_bindings/docs/source/release/12.X.Y-notes.rst b/cuda_bindings/docs/source/release/12.X.Y-notes.rst index f5e5d00a172..4d37acb7a2d 100644 --- a/cuda_bindings/docs/source/release/12.X.Y-notes.rst +++ b/cuda_bindings/docs/source/release/12.X.Y-notes.rst @@ -33,6 +33,8 @@ Highlights Bug fixes --------- +* Fixed a crash on free-threaded Python when callbacks (for example ``cuLogsRegisterCallback``) are registered or + unregistered from multiple threads at the same time. Access to the internal callback registry is now synchronized. Miscellaneous ------------- diff --git a/cuda_bindings/tests/test_callback_threads.py b/cuda_bindings/tests/test_callback_threads.py new file mode 100644 index 00000000000..33659c99eaa --- /dev/null +++ b/cuda_bindings/tests/test_callback_threads.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import threading + +import pytest + +from cuda.bindings import driver + +NUM_THREADS = 16 +NUM_ITERATIONS = 500 + + +@pytest.mark.agent_authored(model="claude-sonnet-5-5") +def test_culogs_register_unregister_from_many_threads(): + (err,) = driver.cuInit(0) + assert err == driver.CUresult.CUDA_SUCCESS + err, version = driver.cuDriverGetVersion() + assert err == driver.CUresult.CUDA_SUCCESS + if version < 12090: + pytest.skip("cuLogs* functions were introduced in CUDA 12.9") + + barrier = threading.Barrier(NUM_THREADS) + failures = [] + + def worker(): + try: + barrier.wait() + for _ in range(NUM_ITERATIONS): + err, handle = driver.cuLogsRegisterCallback(0, None) + assert err == driver.CUresult.CUDA_SUCCESS + (err,) = driver.cuLogsUnregisterCallback(handle) + assert err == driver.CUresult.CUDA_SUCCESS + except BaseException as e: + failures.append(e) + + threads = [threading.Thread(target=worker) for _ in range(NUM_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not failures, failures From 24eeaf505dc8401c408150a1a223c9bfc4e65946 Mon Sep 17 00:00:00 2001 From: Jinfeng Date: Thu, 5 Nov 2026 02:22:36 +0000 Subject: [PATCH 2/5] replace cuLog* with cuDeviceRegister* due to cuLog bug still existed in 12.9 --- cuda_bindings/tests/test_callback_threads.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cuda_bindings/tests/test_callback_threads.py b/cuda_bindings/tests/test_callback_threads.py index 33659c99eaa..1c0f44f2c8a 100644 --- a/cuda_bindings/tests/test_callback_threads.py +++ b/cuda_bindings/tests/test_callback_threads.py @@ -12,13 +12,11 @@ @pytest.mark.agent_authored(model="claude-sonnet-5-5") -def test_culogs_register_unregister_from_many_threads(): +def test_async_notification_register_unregister_from_many_threads(): (err,) = driver.cuInit(0) assert err == driver.CUresult.CUDA_SUCCESS - err, version = driver.cuDriverGetVersion() + err, device = driver.cuDeviceGet(0) assert err == driver.CUresult.CUDA_SUCCESS - if version < 12090: - pytest.skip("cuLogs* functions were introduced in CUDA 12.9") barrier = threading.Barrier(NUM_THREADS) failures = [] @@ -27,9 +25,9 @@ def worker(): try: barrier.wait() for _ in range(NUM_ITERATIONS): - err, handle = driver.cuLogsRegisterCallback(0, None) + err, handle = driver.cuDeviceRegisterAsyncNotification(device, 0, 0) assert err == driver.CUresult.CUDA_SUCCESS - (err,) = driver.cuLogsUnregisterCallback(handle) + (err,) = driver.cuDeviceUnregisterAsyncNotification(device, handle) assert err == driver.CUresult.CUDA_SUCCESS except BaseException as e: failures.append(e) From 86320d781ad76ebcb26196f12d7141bd6a851465 Mon Sep 17 00:00:00 2001 From: Jinfeng Date: Fri, 6 Nov 2026 05:32:23 +0000 Subject: [PATCH 3/5] add sha256 hashes --- cuda_bindings/cuda/bindings/driver.pyx | 2 +- cuda_bindings/cuda/bindings/runtime.pyx.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cuda_bindings/cuda/bindings/driver.pyx b/cuda_bindings/cuda/bindings/driver.pyx index bb1fb3b27c0..d679cd5c964 100644 --- a/cuda_bindings/cuda/bindings/driver.pyx +++ b/cuda_bindings/cuda/bindings/driver.pyx @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This code was automatically generated with version 12.9.0. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=7bd998915de52155f2e27696c9a4d514071b9c721d94e05f0a00c7a07394a961 +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=52f8f192ef10bb489857ef7efde68c518ee854b6b1922ad9492f0d96fb1eac28 from typing import Any, Optional import cython import ctypes diff --git a/cuda_bindings/cuda/bindings/runtime.pyx.in b/cuda_bindings/cuda/bindings/runtime.pyx.in index aa6711d3a3b..c95a9efe561 100644 --- a/cuda_bindings/cuda/bindings/runtime.pyx.in +++ b/cuda_bindings/cuda/bindings/runtime.pyx.in @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This code was automatically generated with version 12.9.0. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=e8e09d2b9dc5c50f5e7a6d85f94bd57c0c0776668393328be79deebdefce1c11 +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=f8e9899c4f71fdba2ecb42e39bdfdbc4fe30e61dbb988cd3d85b64fdba157099 from typing import Any, Optional import cython import ctypes From 5d337daa4551498a710e14aed1b281846caa9112 Mon Sep 17 00:00:00 2001 From: Jinfeng Date: Thu, 12 Nov 2026 21:58:32 +0000 Subject: [PATCH 4/5] regenerate cuda bindings 12.9.x --- cuda_bindings/cuda/bindings/driver.pyx | 4 ++-- cuda_bindings/cuda/bindings/runtime.pyx | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cuda_bindings/cuda/bindings/driver.pyx b/cuda_bindings/cuda/bindings/driver.pyx index c625773b038..b2208732f35 100644 --- a/cuda_bindings/cuda/bindings/driver.pyx +++ b/cuda_bindings/cuda/bindings/driver.pyx @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This code was automatically generated with version 12.9.0. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=67ecc286b7f346fb9026fcdb8fb0914b5d44e31a61570a0f9733513db0a4c81a +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=e99438cba367ec2bee3420a771067ba7d33770c5c4cb189b770f59d56e8ace8c from typing import Any, Optional import cython import ctypes @@ -22160,7 +22160,7 @@ cdef class anon_union15: def pArray(self): return [CUarray(init_value=_pArray) for _pArray in self._pvt_ptr[0].frame.pArray] @pArray.setter - def pArray(self, pArray : list[CUarray]): + def pArray(self, pArray : list[CUarray | int]): if len(pArray) != 3: raise IndexError('not enough values found during array assignment, expected 3, got', len(pArray)) pArray = [int(_pArray) for _pArray in pArray] diff --git a/cuda_bindings/cuda/bindings/runtime.pyx b/cuda_bindings/cuda/bindings/runtime.pyx index 11dfa065114..215191ef316 100644 --- a/cuda_bindings/cuda/bindings/runtime.pyx +++ b/cuda_bindings/cuda/bindings/runtime.pyx @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This code was automatically generated with version 12.9.0. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=43dd07707817f1747c7d5de898f7c687bbea9d20f1c69bd419389a80a432049c +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=be66dc3db6a8e66386b35398ce32c2a7f5edc02e1aa834f0f3c1c77c20d8d8e3 from typing import Any, Optional import cython import ctypes @@ -216,6 +216,18 @@ cudaInvalidDeviceId = cyruntime.cudaInvalidDeviceId #: call cudaInitDeviceFlagsAreValid = cyruntime.cudaInitDeviceFlagsAreValid +#: If set, each kernel launched as part of +#: :py:obj:`~.cudaLaunchCooperativeKernelMultiDevice` only waits for prior +#: work in the stream corresponding to that GPU to complete before the +#: kernel begins execution. +cudaCooperativeLaunchMultiDeviceNoPreSync = cyruntime.cudaCooperativeLaunchMultiDeviceNoPreSync + +#: If set, any subsequent work pushed in a stream that participated in a +#: call to :py:obj:`~.cudaLaunchCooperativeKernelMultiDevice` will only +#: wait for the kernel launched on the GPU corresponding to that stream to +#: complete before it begins execution. +cudaCooperativeLaunchMultiDeviceNoPostSync = cyruntime.cudaCooperativeLaunchMultiDeviceNoPostSync + #: Indicates that the layered sparse CUDA array or CUDA mipmapped array has #: a single mip tail region for all layers cudaArraySparsePropertiesSingleMipTail = cyruntime.cudaArraySparsePropertiesSingleMipTail @@ -17791,7 +17803,7 @@ cdef class anon_union11: def pArray(self): return [cudaArray_t(init_value=_pArray) for _pArray in self._pvt_ptr[0].frame.pArray] @pArray.setter - def pArray(self, pArray : list[cudaArray_t]): + def pArray(self, pArray : list[cudaArray_t | int]): if len(pArray) != 3: raise IndexError('not enough values found during array assignment, expected 3, got', len(pArray)) pArray = [int(_pArray) for _pArray in pArray] From 45e491873fdc2d8cade396a2393c12fedc5f55af Mon Sep 17 00:00:00 2001 From: Jinfeng Date: Thu, 12 Nov 2026 22:25:39 +0000 Subject: [PATCH 5/5] revert unrelated regeneration drift in driver.pyx and runtime.pyx --- cuda_bindings/cuda/bindings/driver.pyx | 4 ++-- cuda_bindings/cuda/bindings/runtime.pyx | 16 ++-------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/cuda_bindings/cuda/bindings/driver.pyx b/cuda_bindings/cuda/bindings/driver.pyx index b2208732f35..c625773b038 100644 --- a/cuda_bindings/cuda/bindings/driver.pyx +++ b/cuda_bindings/cuda/bindings/driver.pyx @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This code was automatically generated with version 12.9.0. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=e99438cba367ec2bee3420a771067ba7d33770c5c4cb189b770f59d56e8ace8c +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=67ecc286b7f346fb9026fcdb8fb0914b5d44e31a61570a0f9733513db0a4c81a from typing import Any, Optional import cython import ctypes @@ -22160,7 +22160,7 @@ cdef class anon_union15: def pArray(self): return [CUarray(init_value=_pArray) for _pArray in self._pvt_ptr[0].frame.pArray] @pArray.setter - def pArray(self, pArray : list[CUarray | int]): + def pArray(self, pArray : list[CUarray]): if len(pArray) != 3: raise IndexError('not enough values found during array assignment, expected 3, got', len(pArray)) pArray = [int(_pArray) for _pArray in pArray] diff --git a/cuda_bindings/cuda/bindings/runtime.pyx b/cuda_bindings/cuda/bindings/runtime.pyx index 215191ef316..11dfa065114 100644 --- a/cuda_bindings/cuda/bindings/runtime.pyx +++ b/cuda_bindings/cuda/bindings/runtime.pyx @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This code was automatically generated with version 12.9.0. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=be66dc3db6a8e66386b35398ce32c2a7f5edc02e1aa834f0f3c1c77c20d8d8e3 +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=43dd07707817f1747c7d5de898f7c687bbea9d20f1c69bd419389a80a432049c from typing import Any, Optional import cython import ctypes @@ -216,18 +216,6 @@ cudaInvalidDeviceId = cyruntime.cudaInvalidDeviceId #: call cudaInitDeviceFlagsAreValid = cyruntime.cudaInitDeviceFlagsAreValid -#: If set, each kernel launched as part of -#: :py:obj:`~.cudaLaunchCooperativeKernelMultiDevice` only waits for prior -#: work in the stream corresponding to that GPU to complete before the -#: kernel begins execution. -cudaCooperativeLaunchMultiDeviceNoPreSync = cyruntime.cudaCooperativeLaunchMultiDeviceNoPreSync - -#: If set, any subsequent work pushed in a stream that participated in a -#: call to :py:obj:`~.cudaLaunchCooperativeKernelMultiDevice` will only -#: wait for the kernel launched on the GPU corresponding to that stream to -#: complete before it begins execution. -cudaCooperativeLaunchMultiDeviceNoPostSync = cyruntime.cudaCooperativeLaunchMultiDeviceNoPostSync - #: Indicates that the layered sparse CUDA array or CUDA mipmapped array has #: a single mip tail region for all layers cudaArraySparsePropertiesSingleMipTail = cyruntime.cudaArraySparsePropertiesSingleMipTail @@ -17803,7 +17791,7 @@ cdef class anon_union11: def pArray(self): return [cudaArray_t(init_value=_pArray) for _pArray in self._pvt_ptr[0].frame.pArray] @pArray.setter - def pArray(self, pArray : list[cudaArray_t | int]): + def pArray(self, pArray : list[cudaArray_t]): if len(pArray) != 3: raise IndexError('not enough values found during array assignment, expected 3, got', len(pArray)) pArray = [int(_pArray) for _pArray in pArray]