From f3642ae527734d5bc41ea6f09358562eff5a1fc1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 08:29:16 +0000 Subject: [PATCH 01/10] fix thread safety --- Lib/test/test_asyncio/test_free_threading.py | 26 + Modules/_asynciomodule.c | 363 ++++++++---- Modules/clinic/_asynciomodule.c.h | 550 ++++++++++++++++++- 3 files changed, 813 insertions(+), 126 deletions(-) create mode 100644 Lib/test/test_asyncio/test_free_threading.py diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py new file mode 100644 index 00000000000000..516a9e57583ce1 --- /dev/null +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -0,0 +1,26 @@ +import asyncio +from test.support import threading_helper +from unittest import TestCase +from threading import Thread + +threading_helper.requires_working_threading(module=True) + +class TestFreeThreading(TestCase): + def test_all_tasks_race(self) -> None: + async def task(): + await asyncio.sleep(0) + + async def main(): + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tg.create_task(task()) + asyncio.all_tasks() + + threads = [] + for _ in range(10): + thread = Thread(target=lambda: asyncio.run(main())) + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 27c16364457336..dfea61df2876c4 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -798,6 +798,7 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) } /*[clinic input] +@critical_section _asyncio.Future.result Return the result this future represents. @@ -809,7 +810,7 @@ the future is done and has an exception set, this exception is raised. static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=49ecf9cf5ec50dc5]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=61d89f48e4c8b670]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -838,6 +839,7 @@ _asyncio_Future_result_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.exception cls: defining_class @@ -853,7 +855,7 @@ InvalidStateError. static PyObject * _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=ce75576b187c905b input=3faf15c22acdb60d]*/ +/*[clinic end generated code: output=ce75576b187c905b input=647d1fd1fc403301]*/ { if (!future_is_alive(self)) { asyncio_state *state = get_asyncio_state_by_cls(cls); @@ -884,6 +886,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _asyncio.Future.set_result cls: defining_class @@ -899,7 +902,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, PyObject *result) -/*[clinic end generated code: output=99afbbe78f99c32d input=d5a41c1e353acc2e]*/ +/*[clinic end generated code: output=99afbbe78f99c32d input=4069306f03a3b6ee]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -907,6 +910,7 @@ _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.set_exception cls: defining_class @@ -922,7 +926,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, PyObject *exception) -/*[clinic end generated code: output=0a5e8b5a52f058d6 input=a245cd49d3df939b]*/ +/*[clinic end generated code: output=0a5e8b5a52f058d6 input=b6eab43a389bc966]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -930,6 +934,7 @@ _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.add_done_callback cls: defining_class @@ -948,7 +953,7 @@ scheduled with call_soon. static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=599261c521458cc2]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=37d97f941beb7b3e]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -964,6 +969,7 @@ _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.remove_done_callback cls: defining_class @@ -978,7 +984,7 @@ Returns the number of callbacks removed. static PyObject * _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn) -/*[clinic end generated code: output=2da35ccabfe41b98 input=c7518709b86fc747]*/ +/*[clinic end generated code: output=2da35ccabfe41b98 input=3afbc9f6a673091b]*/ { PyObject *newlist; Py_ssize_t len, i, j=0; @@ -1087,6 +1093,7 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.cancel cls: defining_class @@ -1103,7 +1110,7 @@ return True. static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=bba8f8b786941a94]*/ +/*[clinic end generated code: output=074956f35904b034 input=44ab4003da839970]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1111,6 +1118,7 @@ _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.cancelled Return True if the future was cancelled. @@ -1118,7 +1126,7 @@ Return True if the future was cancelled. static PyObject * _asyncio_Future_cancelled_impl(FutureObj *self) -/*[clinic end generated code: output=145197ced586357d input=943ab8b7b7b17e45]*/ +/*[clinic end generated code: output=145197ced586357d input=9b8644819a675416]*/ { if (future_is_alive(self) && self->fut_state == STATE_CANCELLED) { Py_RETURN_TRUE; @@ -1129,6 +1137,7 @@ _asyncio_Future_cancelled_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.done Return True if the future is done. @@ -1139,7 +1148,7 @@ future was cancelled. static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=28d7b23fdb65d2ac]*/ +/*[clinic end generated code: output=244c5ac351145096 input=7204d3cc63bef7f3]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1150,6 +1159,7 @@ _asyncio_Future_done_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.get_loop cls: defining_class @@ -1160,17 +1170,24 @@ Return the event loop the Future is bound to. static PyObject * _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=f50ea6c374d9ee97 input=163c2c498b45a1f0]*/ +/*[clinic end generated code: output=f50ea6c374d9ee97 input=f3ce629bfd9f45c1]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) return Py_NewRef(self->fut_loop); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._asyncio_future_blocking +[clinic start generated code]*/ + static PyObject * -FutureObj_get_blocking(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__asyncio_future_blocking_get_impl(FutureObj *self) +/*[clinic end generated code: output=a558a2c51e38823b input=58da92efc03b617d]*/ { - if (future_is_alive(fut) && fut->fut_blocking) { + if (future_is_alive(self) && self->fut_blocking) { Py_RETURN_TRUE; } else { @@ -1178,31 +1195,47 @@ FutureObj_get_blocking(FutureObj *fut, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._asyncio_future_blocking +[clinic start generated code]*/ + static int -FutureObj_set_blocking(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Future__asyncio_future_blocking_set_impl(FutureObj *self, + PyObject *value) +/*[clinic end generated code: output=0686d1cb024a7453 input=3fd4a5f95df788b7]*/ + { - if (future_ensure_alive(fut)) { + if (future_ensure_alive(self)) { return -1; } - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } - fut->fut_blocking = is_true; + self->fut_blocking = is_true; return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._log_traceback +[clinic start generated code]*/ + static PyObject * -FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__log_traceback_get_impl(FutureObj *self) +/*[clinic end generated code: output=2724433b238593c7 input=91e5144ea4117d8e]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_log_tb) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_log_tb) { Py_RETURN_TRUE; } else { @@ -1210,14 +1243,21 @@ FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._log_traceback +[clinic start generated code]*/ + static int -FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Future__log_traceback_set_impl(FutureObj *self, PyObject *value) +/*[clinic end generated code: output=9ce8e19504f42f54 input=30ac8217754b08c2]*/ { - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } @@ -1226,31 +1266,44 @@ FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignor "_log_traceback can only be set to False"); return -1; } - fut->fut_log_tb = is_true; + self->fut_log_tb = is_true; return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._loop +[clinic start generated code]*/ static PyObject * -FutureObj_get_loop(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__loop_get_impl(FutureObj *self) +/*[clinic end generated code: output=5ba31563eecfeedf input=0337130bc5781670]*/ { - if (!future_is_alive(fut)) { + if (!future_is_alive(self)) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_loop); + return Py_NewRef(self->fut_loop); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._callbacks +[clinic start generated code]*/ + static PyObject * -FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__callbacks_get_impl(FutureObj *self) +/*[clinic end generated code: output=b40d360505fcc583 input=7a466649530c01bb]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) Py_ssize_t len = 0; - if (fut->fut_callback0 != NULL) { + if (self->fut_callback0 != NULL) { len++; } - if (fut->fut_callbacks != NULL) { - len += PyList_GET_SIZE(fut->fut_callbacks); + if (self->fut_callbacks != NULL) { + len += PyList_GET_SIZE(self->fut_callbacks); } if (len == 0) { @@ -1263,22 +1316,22 @@ FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) } Py_ssize_t i = 0; - if (fut->fut_callback0 != NULL) { + if (self->fut_callback0 != NULL) { PyObject *tup0 = PyTuple_New(2); if (tup0 == NULL) { Py_DECREF(callbacks); return NULL; } - PyTuple_SET_ITEM(tup0, 0, Py_NewRef(fut->fut_callback0)); - assert(fut->fut_context0 != NULL); - PyTuple_SET_ITEM(tup0, 1, Py_NewRef(fut->fut_context0)); + PyTuple_SET_ITEM(tup0, 0, Py_NewRef(self->fut_callback0)); + assert(self->fut_context0 != NULL); + PyTuple_SET_ITEM(tup0, 1, Py_NewRef(self->fut_context0)); PyList_SET_ITEM(callbacks, i, tup0); i++; } - if (fut->fut_callbacks != NULL) { - for (Py_ssize_t j = 0; j < PyList_GET_SIZE(fut->fut_callbacks); j++) { - PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, j); + if (self->fut_callbacks != NULL) { + for (Py_ssize_t j = 0; j < PyList_GET_SIZE(self->fut_callbacks); j++) { + PyObject *cb = PyList_GET_ITEM(self->fut_callbacks, j); Py_INCREF(cb); PyList_SET_ITEM(callbacks, i, cb); i++; @@ -1288,68 +1341,110 @@ FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) return callbacks; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._result +[clinic start generated code]*/ + static PyObject * -FutureObj_get_result(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__result_get_impl(FutureObj *self) +/*[clinic end generated code: output=6877e8ce97333873 input=624f8e28e67f2636]*/ + { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_result == NULL) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_result == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_result); + return Py_NewRef(self->fut_result); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._exception +[clinic start generated code]*/ + static PyObject * -FutureObj_get_exception(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__exception_get_impl(FutureObj *self) +/*[clinic end generated code: output=32f2c93b9e021a9b input=1828a1fcac929710]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_exception == NULL) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_exception == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_exception); + return Py_NewRef(self->fut_exception); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._source_traceback +[clinic start generated code]*/ + static PyObject * -FutureObj_get_source_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__source_traceback_get_impl(FutureObj *self) +/*[clinic end generated code: output=d4f12b09af22f61b input=3c831fbde5da90d0]*/ { - if (!future_is_alive(fut) || fut->fut_source_tb == NULL) { + if (!future_is_alive(self) || self->fut_source_tb == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_source_tb); + return Py_NewRef(self->fut_source_tb); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._cancel_message +[clinic start generated code]*/ + static PyObject * -FutureObj_get_cancel_message(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__cancel_message_get_impl(FutureObj *self) +/*[clinic end generated code: output=52ef6444f92cedac input=54c12c67082e4eea]*/ { - if (fut->fut_cancel_msg == NULL) { + if (self->fut_cancel_msg == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_cancel_msg); + return Py_NewRef(self->fut_cancel_msg); } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._cancel_message +[clinic start generated code]*/ + static int -FutureObj_set_cancel_message(FutureObj *fut, PyObject *msg, - void *Py_UNUSED(ignored)) +_asyncio_Future__cancel_message_set_impl(FutureObj *self, PyObject *value) +/*[clinic end generated code: output=0854b2f77bff2209 input=f461d17f2d891fad]*/ { - if (msg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - Py_INCREF(msg); - Py_XSETREF(fut->fut_cancel_msg, msg); + Py_INCREF(value); + Py_XSETREF(self->fut_cancel_msg, value); return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._state +[clinic start generated code]*/ + static PyObject * -FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__state_get_impl(FutureObj *self) +/*[clinic end generated code: output=622f560a3fa69c63 input=7c5ad023a93423ff]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *ret = NULL; - ENSURE_FUTURE_ALIVE(state, fut) + ENSURE_FUTURE_ALIVE(state, self) - switch (fut->fut_state) { + switch (self->fut_state) { case STATE_PENDING: ret = &_Py_ID(PENDING); break; @@ -1375,6 +1470,7 @@ FutureObj_repr(FutureObj *fut) } /*[clinic input] +@critical_section _asyncio.Future._make_cancelled_error Create the CancelledError to raise if the Future is cancelled. @@ -1385,7 +1481,7 @@ it erases the context exception value. static PyObject * _asyncio_Future__make_cancelled_error_impl(FutureObj *self) -/*[clinic end generated code: output=a5df276f6c1213de input=ac6effe4ba795ecc]*/ +/*[clinic end generated code: output=a5df276f6c1213de input=ccb90df8c3c18bcd]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); return create_cancelled_error(state, self); @@ -1466,23 +1562,16 @@ static PyMethodDef FutureType_methods[] = { {NULL, NULL} /* Sentinel */ }; -#define FUTURE_COMMON_GETSETLIST \ - {"_state", (getter)FutureObj_get_state, NULL, NULL}, \ - {"_asyncio_future_blocking", (getter)FutureObj_get_blocking, \ - (setter)FutureObj_set_blocking, NULL}, \ - {"_loop", (getter)FutureObj_get_loop, NULL, NULL}, \ - {"_callbacks", (getter)FutureObj_get_callbacks, NULL, NULL}, \ - {"_result", (getter)FutureObj_get_result, NULL, NULL}, \ - {"_exception", (getter)FutureObj_get_exception, NULL, NULL}, \ - {"_log_traceback", (getter)FutureObj_get_log_traceback, \ - (setter)FutureObj_set_log_traceback, NULL}, \ - {"_source_traceback", (getter)FutureObj_get_source_traceback, \ - NULL, NULL}, \ - {"_cancel_message", (getter)FutureObj_get_cancel_message, \ - (setter)FutureObj_set_cancel_message, NULL}, - static PyGetSetDef FutureType_getsetlist[] = { - FUTURE_COMMON_GETSETLIST + _ASYNCIO_FUTURE__STATE_GETSETDEF + _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF + _ASYNCIO_FUTURE__LOOP_GETSETDEF + _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF + _ASYNCIO_FUTURE__RESULT_GETSETDEF + _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF + _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF + _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF + _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {NULL} /* Sentinel */ }; @@ -2174,10 +2263,17 @@ TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Task._log_destroy_pending +[clinic start generated code]*/ + static PyObject * -TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__log_destroy_pending_get_impl(TaskObj *self) +/*[clinic end generated code: output=e6c2a47d029ac93b input=17127298cd4c720b]*/ { - if (task->task_log_destroy_pending) { + if (self->task_log_destroy_pending) { Py_RETURN_TRUE; } else { @@ -2185,25 +2281,40 @@ TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Task._log_destroy_pending +[clinic start generated code]*/ + static int -TaskObj_set_log_destroy_pending(TaskObj *task, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Task__log_destroy_pending_set_impl(TaskObj *self, PyObject *value) +/*[clinic end generated code: output=7ebc030bb92ec5ce input=49b759c97d1216a4]*/ { - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } - task->task_log_destroy_pending = is_true; + self->task_log_destroy_pending = is_true; return 0; } + +/*[clinic input] +@critical_section +@getter +_asyncio.Task._must_cancel +[clinic start generated code]*/ + static PyObject * -TaskObj_get_must_cancel(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__must_cancel_get_impl(TaskObj *self) +/*[clinic end generated code: output=70e79b900996c363 input=2d04529fb23feedf]*/ { - if (task->task_must_cancel) { + if (self->task_must_cancel) { Py_RETURN_TRUE; } else { @@ -2211,21 +2322,36 @@ TaskObj_get_must_cancel(TaskObj *task, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@getter +_asyncio.Task._coro +[clinic start generated code]*/ + static PyObject * -TaskObj_get_coro(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__coro_get_impl(TaskObj *self) +/*[clinic end generated code: output=a2726012ab5fd531 input=323c31a272020624]*/ { - if (task->task_coro) { - return Py_NewRef(task->task_coro); + if (self->task_coro) { + return Py_NewRef(self->task_coro); } Py_RETURN_NONE; } + +/*[clinic input] +@critical_section +@getter +_asyncio.Task._fut_waiter +[clinic start generated code]*/ + static PyObject * -TaskObj_get_fut_waiter(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__fut_waiter_get_impl(TaskObj *self) +/*[clinic end generated code: output=c4f966b847fefcdf input=4d1005d725e72db7]*/ { - if (task->task_fut_waiter) { - return Py_NewRef(task->task_fut_waiter); + if (self->task_fut_waiter) { + return Py_NewRef(self->task_fut_waiter); } Py_RETURN_NONE; @@ -2241,6 +2367,7 @@ TaskObj_repr(TaskObj *task) /*[clinic input] +@critical_section _asyncio.Task._make_cancelled_error Create the CancelledError to raise if the Task is cancelled. @@ -2251,7 +2378,7 @@ it erases the context exception value. static PyObject * _asyncio_Task__make_cancelled_error_impl(TaskObj *self) -/*[clinic end generated code: output=55a819e8b4276fab input=52c0e32de8e2f840]*/ +/*[clinic end generated code: output=55a819e8b4276fab input=2d3213be0cb02390]*/ { FutureObj *fut = (FutureObj*)self; return _asyncio_Future__make_cancelled_error_impl(fut); @@ -2259,6 +2386,7 @@ _asyncio_Task__make_cancelled_error_impl(TaskObj *self) /*[clinic input] +@critical_section _asyncio.Task.cancel msg: object = None @@ -2287,7 +2415,7 @@ This also increases the task's count of cancellation requests. static PyObject * _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg) -/*[clinic end generated code: output=c66b60d41c74f9f1 input=7bb51bf25974c783]*/ +/*[clinic end generated code: output=c66b60d41c74f9f1 input=6125d45b9a6a5abd]*/ { self->task_log_tb = 0; @@ -2332,6 +2460,7 @@ _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg) } /*[clinic input] +@critical_section _asyncio.Task.cancelling Return the count of the task's cancellation requests. @@ -2342,13 +2471,14 @@ and may be decremented using .uncancel(). static PyObject * _asyncio_Task_cancelling_impl(TaskObj *self) -/*[clinic end generated code: output=803b3af96f917d7e input=b625224d310cbb17]*/ +/*[clinic end generated code: output=803b3af96f917d7e input=5ef89b1b38f080ee]*/ /*[clinic end generated code]*/ { return PyLong_FromLong(self->task_num_cancels_requested); } /*[clinic input] +@critical_section _asyncio.Task.uncancel Decrement the task's count of cancellation requests. @@ -2361,7 +2491,7 @@ Returns the remaining number of cancellation requests. static PyObject * _asyncio_Task_uncancel_impl(TaskObj *self) -/*[clinic end generated code: output=58184d236a817d3c input=68f81a4b90b46be2]*/ +/*[clinic end generated code: output=58184d236a817d3c input=cb3220b0e5afd61d]*/ /*[clinic end generated code]*/ { if (self->task_num_cancels_requested > 0) { @@ -2475,12 +2605,13 @@ _asyncio_Task_set_exception(TaskObj *self, PyObject *exception) } /*[clinic input] +@critical_section _asyncio.Task.get_coro [clinic start generated code]*/ static PyObject * _asyncio_Task_get_coro_impl(TaskObj *self) -/*[clinic end generated code: output=bcac27c8cc6c8073 input=d2e8606c42a7b403]*/ +/*[clinic end generated code: output=bcac27c8cc6c8073 input=a47f81427e39fe0c]*/ { if (self->task_coro) { return Py_NewRef(self->task_coro); @@ -2501,12 +2632,13 @@ _asyncio_Task_get_context_impl(TaskObj *self) } /*[clinic input] +@critical_section _asyncio.Task.get_name [clinic start generated code]*/ static PyObject * _asyncio_Task_get_name_impl(TaskObj *self) -/*[clinic end generated code: output=0ecf1570c3b37a8f input=a4a6595d12f4f0f8]*/ +/*[clinic end generated code: output=0ecf1570c3b37a8f input=92a8f30c85034249]*/ { if (self->task_name) { if (PyLong_CheckExact(self->task_name)) { @@ -2523,6 +2655,7 @@ _asyncio_Task_get_name_impl(TaskObj *self) } /*[clinic input] +@critical_section _asyncio.Task.set_name value: object @@ -2530,8 +2663,8 @@ _asyncio.Task.set_name [clinic start generated code]*/ static PyObject * -_asyncio_Task_set_name(TaskObj *self, PyObject *value) -/*[clinic end generated code: output=138a8d51e32057d6 input=a8359b6e65f8fd31]*/ +_asyncio_Task_set_name_impl(TaskObj *self, PyObject *value) +/*[clinic end generated code: output=f88ff4c0d64a9a6f input=e8d400ad64bad799]*/ { if (!PyUnicode_CheckExact(value)) { value = PyObject_Str(value); @@ -2642,12 +2775,10 @@ static PyMethodDef TaskType_methods[] = { }; static PyGetSetDef TaskType_getsetlist[] = { - FUTURE_COMMON_GETSETLIST - {"_log_destroy_pending", (getter)TaskObj_get_log_destroy_pending, - (setter)TaskObj_set_log_destroy_pending, NULL}, - {"_must_cancel", (getter)TaskObj_get_must_cancel, NULL, NULL}, - {"_coro", (getter)TaskObj_get_coro, NULL, NULL}, - {"_fut_waiter", (getter)TaskObj_get_fut_waiter, NULL, NULL}, + _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF + _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF + _ASYNCIO_TASK__CORO_GETSETDEF + _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {NULL} /* Sentinel */ }; @@ -3605,7 +3736,15 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) } // First add eager tasks to the set so that we don't miss // any tasks which graduates from eager to non-eager - PyObject *eager_iter = PyObject_GetIter(state->eager_tasks); + // Makes a copy of set before iterating to avoid + // mutations from other threads. + PyObject *eager_tasks = PySet_New(state->eager_tasks); + if (eager_tasks == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + PyObject *eager_iter = PyObject_GetIter(eager_tasks); if (eager_iter == NULL) { Py_DECREF(tasks); Py_DECREF(loop); diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 32045804c35004..3a37cdd9b5fa83 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_asyncio_Future___init____doc__, @@ -98,7 +99,13 @@ _asyncio_Future_result_impl(FutureObj *self); static PyObject * _asyncio_Future_result(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_result_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_result_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_exception__doc__, @@ -121,11 +128,18 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); - return NULL; + goto exit; } - return _asyncio_Future_exception_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_exception_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_asyncio_Future_set_result__doc__, @@ -170,7 +184,9 @@ _asyncio_Future_set_result(FutureObj *self, PyTypeObject *cls, PyObject *const * goto exit; } result = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_set_result_impl(self, cls, result); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -218,7 +234,9 @@ _asyncio_Future_set_exception(FutureObj *self, PyTypeObject *cls, PyObject *cons goto exit; } exception = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_set_exception_impl(self, cls, exception); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -286,7 +304,9 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject * } context = args[1]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_add_done_callback_impl(self, cls, fn, context); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -333,7 +353,9 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyTypeObject *cls, PyObjec goto exit; } fn = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_remove_done_callback_impl(self, cls, fn); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -399,7 +421,9 @@ _asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args } msg = args[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_cancel_impl(self, cls, msg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -420,7 +444,13 @@ _asyncio_Future_cancelled_impl(FutureObj *self); static PyObject * _asyncio_Future_cancelled(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_cancelled_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_cancelled_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_done__doc__, @@ -441,7 +471,13 @@ _asyncio_Future_done_impl(FutureObj *self); static PyObject * _asyncio_Future_done(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_done_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_done_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_get_loop__doc__, @@ -459,11 +495,319 @@ _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); - return NULL; + goto exit; } - return _asyncio_Future_get_loop_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_get_loop_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if !defined(_asyncio_Future__asyncio_future_blocking_DOCSTR) +# define _asyncio_Future__asyncio_future_blocking_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, (setter)_asyncio_Future__asyncio_future_blocking_set, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, NULL, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__asyncio_future_blocking_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__asyncio_future_blocking_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_future_blocking_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__asyncio_future_blocking_DOCSTR) +# define _asyncio_Future__asyncio_future_blocking_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, (setter)_asyncio_Future__asyncio_future_blocking_set, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", NULL, (setter)_asyncio_Future__asyncio_future_blocking_set, NULL}, +#endif + +static int +_asyncio_Future__asyncio_future_blocking_set_impl(FutureObj *self, + PyObject *value); + +static int +_asyncio_Future__asyncio_future_blocking_set(FutureObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_future_blocking_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__log_traceback_DOCSTR) +# define _asyncio_Future__log_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, (setter)_asyncio_Future__log_traceback_set, _asyncio_Future__log_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, NULL, _asyncio_Future__log_traceback_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__log_traceback_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__log_traceback_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__log_traceback_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__log_traceback_DOCSTR) +# define _asyncio_Future__log_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, (setter)_asyncio_Future__log_traceback_set, _asyncio_Future__log_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", NULL, (setter)_asyncio_Future__log_traceback_set, NULL}, +#endif + +static int +_asyncio_Future__log_traceback_set_impl(FutureObj *self, PyObject *value); + +static int +_asyncio_Future__log_traceback_set(FutureObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__log_traceback_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__loop_DOCSTR) +# define _asyncio_Future__loop_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOOP_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOOP_GETSETDEF +# define _ASYNCIO_FUTURE__LOOP_GETSETDEF {"_loop", (getter)_asyncio_Future__loop_get, (setter)_asyncio_Future__loop_set, _asyncio_Future__loop_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOOP_GETSETDEF {"_loop", (getter)_asyncio_Future__loop_get, NULL, _asyncio_Future__loop_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__loop_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__loop_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__loop_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__callbacks_DOCSTR) +# define _asyncio_Future__callbacks_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CALLBACKS_GETSETDEF) +# undef _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF +# define _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF {"_callbacks", (getter)_asyncio_Future__callbacks_get, (setter)_asyncio_Future__callbacks_set, _asyncio_Future__callbacks_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF {"_callbacks", (getter)_asyncio_Future__callbacks_get, NULL, _asyncio_Future__callbacks_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__callbacks_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__callbacks_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__callbacks_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__result_DOCSTR) +# define _asyncio_Future__result_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__RESULT_GETSETDEF) +# undef _ASYNCIO_FUTURE__RESULT_GETSETDEF +# define _ASYNCIO_FUTURE__RESULT_GETSETDEF {"_result", (getter)_asyncio_Future__result_get, (setter)_asyncio_Future__result_set, _asyncio_Future__result_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__RESULT_GETSETDEF {"_result", (getter)_asyncio_Future__result_get, NULL, _asyncio_Future__result_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__result_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__result_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__result_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__exception_DOCSTR) +# define _asyncio_Future__exception_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__EXCEPTION_GETSETDEF) +# undef _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF +# define _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF {"_exception", (getter)_asyncio_Future__exception_get, (setter)_asyncio_Future__exception_set, _asyncio_Future__exception_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF {"_exception", (getter)_asyncio_Future__exception_get, NULL, _asyncio_Future__exception_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__exception_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__exception_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__exception_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__source_traceback_DOCSTR) +# define _asyncio_Future__source_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF {"_source_traceback", (getter)_asyncio_Future__source_traceback_get, (setter)_asyncio_Future__source_traceback_set, _asyncio_Future__source_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF {"_source_traceback", (getter)_asyncio_Future__source_traceback_get, NULL, _asyncio_Future__source_traceback_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__source_traceback_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__source_traceback_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__source_traceback_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__cancel_message_DOCSTR) +# define _asyncio_Future__cancel_message_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF) +# undef _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, (setter)_asyncio_Future__cancel_message_set, _asyncio_Future__cancel_message_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, NULL, _asyncio_Future__cancel_message_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__cancel_message_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__cancel_message_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__cancel_message_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__cancel_message_DOCSTR) +# define _asyncio_Future__cancel_message_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF) +# undef _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, (setter)_asyncio_Future__cancel_message_set, _asyncio_Future__cancel_message_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", NULL, (setter)_asyncio_Future__cancel_message_set, NULL}, +#endif + +static int +_asyncio_Future__cancel_message_set_impl(FutureObj *self, PyObject *value); + +static int +_asyncio_Future__cancel_message_set(FutureObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__cancel_message_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__state_DOCSTR) +# define _asyncio_Future__state_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__STATE_GETSETDEF) +# undef _ASYNCIO_FUTURE__STATE_GETSETDEF +# define _ASYNCIO_FUTURE__STATE_GETSETDEF {"_state", (getter)_asyncio_Future__state_get, (setter)_asyncio_Future__state_set, _asyncio_Future__state_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__STATE_GETSETDEF {"_state", (getter)_asyncio_Future__state_get, NULL, _asyncio_Future__state_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__state_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__state_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__state_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future__make_cancelled_error__doc__, @@ -484,7 +828,13 @@ _asyncio_Future__make_cancelled_error_impl(FutureObj *self); static PyObject * _asyncio_Future__make_cancelled_error(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future__make_cancelled_error_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__make_cancelled_error_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task___init____doc__, @@ -575,6 +925,131 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +#if !defined(_asyncio_Task__log_destroy_pending_DOCSTR) +# define _asyncio_Task__log_destroy_pending_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF) +# undef _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, (setter)_asyncio_Task__log_destroy_pending_set, _asyncio_Task__log_destroy_pending_DOCSTR}, +#else +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, NULL, _asyncio_Task__log_destroy_pending_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__log_destroy_pending_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__log_destroy_pending_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__log_destroy_pending_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__log_destroy_pending_DOCSTR) +# define _asyncio_Task__log_destroy_pending_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF) +# undef _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, (setter)_asyncio_Task__log_destroy_pending_set, _asyncio_Task__log_destroy_pending_DOCSTR}, +#else +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", NULL, (setter)_asyncio_Task__log_destroy_pending_set, NULL}, +#endif + +static int +_asyncio_Task__log_destroy_pending_set_impl(TaskObj *self, PyObject *value); + +static int +_asyncio_Task__log_destroy_pending_set(TaskObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__log_destroy_pending_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__must_cancel_DOCSTR) +# define _asyncio_Task__must_cancel_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__MUST_CANCEL_GETSETDEF) +# undef _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF +# define _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF {"_must_cancel", (getter)_asyncio_Task__must_cancel_get, (setter)_asyncio_Task__must_cancel_set, _asyncio_Task__must_cancel_DOCSTR}, +#else +# define _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF {"_must_cancel", (getter)_asyncio_Task__must_cancel_get, NULL, _asyncio_Task__must_cancel_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__must_cancel_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__must_cancel_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__must_cancel_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__coro_DOCSTR) +# define _asyncio_Task__coro_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__CORO_GETSETDEF) +# undef _ASYNCIO_TASK__CORO_GETSETDEF +# define _ASYNCIO_TASK__CORO_GETSETDEF {"_coro", (getter)_asyncio_Task__coro_get, (setter)_asyncio_Task__coro_set, _asyncio_Task__coro_DOCSTR}, +#else +# define _ASYNCIO_TASK__CORO_GETSETDEF {"_coro", (getter)_asyncio_Task__coro_get, NULL, _asyncio_Task__coro_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__coro_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__coro_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__coro_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__fut_waiter_DOCSTR) +# define _asyncio_Task__fut_waiter_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__FUT_WAITER_GETSETDEF) +# undef _ASYNCIO_TASK__FUT_WAITER_GETSETDEF +# define _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {"_fut_waiter", (getter)_asyncio_Task__fut_waiter_get, (setter)_asyncio_Task__fut_waiter_set, _asyncio_Task__fut_waiter_DOCSTR}, +#else +# define _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {"_fut_waiter", (getter)_asyncio_Task__fut_waiter_get, NULL, _asyncio_Task__fut_waiter_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__fut_waiter_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__fut_waiter_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__fut_waiter_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio_Task__make_cancelled_error__doc__, "_make_cancelled_error($self, /)\n" "--\n" @@ -593,7 +1068,13 @@ _asyncio_Task__make_cancelled_error_impl(TaskObj *self); static PyObject * _asyncio_Task__make_cancelled_error(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task__make_cancelled_error_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__make_cancelled_error_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_cancel__doc__, @@ -670,7 +1151,9 @@ _asyncio_Task_cancel(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyO } msg = args[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Task_cancel_impl(self, msg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -694,7 +1177,13 @@ _asyncio_Task_cancelling_impl(TaskObj *self); static PyObject * _asyncio_Task_cancelling(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_cancelling_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_cancelling_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_uncancel__doc__, @@ -717,7 +1206,13 @@ _asyncio_Task_uncancel_impl(TaskObj *self); static PyObject * _asyncio_Task_uncancel(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_uncancel_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_uncancel_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_get_stack__doc__, @@ -905,7 +1400,13 @@ _asyncio_Task_get_coro_impl(TaskObj *self); static PyObject * _asyncio_Task_get_coro(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_coro_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_get_coro_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_get_context__doc__, @@ -939,7 +1440,13 @@ _asyncio_Task_get_name_impl(TaskObj *self); static PyObject * _asyncio_Task_get_name(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_name_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_get_name_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_set_name__doc__, @@ -950,6 +1457,21 @@ PyDoc_STRVAR(_asyncio_Task_set_name__doc__, #define _ASYNCIO_TASK_SET_NAME_METHODDEF \ {"set_name", (PyCFunction)_asyncio_Task_set_name, METH_O, _asyncio_Task_set_name__doc__}, +static PyObject * +_asyncio_Task_set_name_impl(TaskObj *self, PyObject *value); + +static PyObject * +_asyncio_Task_set_name(TaskObj *self, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_set_name_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio__get_running_loop__doc__, "_get_running_loop($module, /)\n" "--\n" @@ -1566,4 +2088,4 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=e5d95a0ec229ffcd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=408e156476ced07f input=a9049054013a1b77]*/ From 1f63b9e0c0166f0d21ec7f8256744980301c4422 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 08:44:25 +0000 Subject: [PATCH 02/10] keep working --- Lib/test/test_asyncio/test_free_threading.py | 3 +++ Modules/_asynciomodule.c | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index 516a9e57583ce1..9d58611348ac77 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -1,3 +1,6 @@ +import sys +sys.modules["_asyncio"] = None + import asyncio from test.support import threading_helper from unittest import TestCase diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index dfea61df2876c4..08608c0508137a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -495,6 +495,7 @@ future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) if (future_ensure_alive(fut)) { return NULL; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); if (fut->fut_state != STATE_PENDING) { PyErr_SetString(state->asyncio_InvalidStateError, "invalid state"); @@ -3254,9 +3255,9 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc) if (enter_task(state, task->task_loop, (PyObject*)task) < 0) { return NULL; } - + Py_BEGIN_CRITICAL_SECTION(task); res = task_step_impl(state, task, exc); - + Py_END_CRITICAL_SECTION(); if (res == NULL) { PyObject *exc = PyErr_GetRaisedException(); leave_task(state, task->task_loop, (PyObject*)task); @@ -3295,7 +3296,10 @@ task_eager_start(asyncio_state *state, TaskObj *task) int retval = 0; - PyObject *stepres = task_step_impl(state, task, NULL); + PyObject *stepres; + Py_BEGIN_CRITICAL_SECTION(task); + stepres = task_step_impl(state, task, NULL); + Py_END_CRITICAL_SECTION(); if (stepres == NULL) { PyObject *exc = PyErr_GetRaisedException(); _PyErr_ChainExceptions1(exc); From 26e9932726b94fc5cd1d4d5817ed5366f220ba23 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 09:38:14 +0000 Subject: [PATCH 03/10] keep working --- Lib/test/test_asyncio/test_free_threading.py | 74 ++++++++++++++++---- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index 9d58611348ac77..e4c28d33e8b280 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -1,29 +1,79 @@ -import sys -sys.modules["_asyncio"] = None - import asyncio -from test.support import threading_helper -from unittest import TestCase +import unittest from threading import Thread +from unittest import TestCase + +from test.support import threading_helper threading_helper.requires_working_threading(module=True) -class TestFreeThreading(TestCase): - def test_all_tasks_race(self) -> None: - async def task(): - await asyncio.sleep(0) +class TestFreeThreading: + def test_all_tasks_race(self) -> None: async def main(): + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def task(): + await future + + tasks = set() + async with asyncio.TaskGroup() as tg: for _ in range(100): - tg.create_task(task()) - asyncio.all_tasks() + tasks.add(tg.create_task(task())) + + all_tasks = self.all_tasks(loop) + self.assertEqual(len(all_tasks), 101) + + for task in tasks: + self.assertEqual(task.get_loop(), loop) + self.assertFalse(task.done()) + + current = self.current_task() + self.assertEqual(current.get_loop(), loop) + self.assertSetEqual(all_tasks, tasks | {current}) + future.set_result(None) + + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(main()) threads = [] + for _ in range(10): - thread = Thread(target=lambda: asyncio.run(main())) + thread = Thread(target=runner) threads.append(thread) with threading_helper.start_threads(threads): pass + +class TestPyFreeThreading(TestFreeThreading, TestCase): + all_tasks = staticmethod(asyncio.tasks._py_all_tasks) + current_task = staticmethod(asyncio.tasks._py_current_task) + + def factory(self, loop, coro, context=None): + return asyncio.tasks._PyTask(coro, loop=loop, context=context) + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestCFreeThreading(TestFreeThreading, TestCase): + all_tasks = staticmethod(getattr(asyncio.tasks, "_c_all_tasks", None)) + current_task = staticmethod(getattr(asyncio.tasks, "_c_current_task", None)) + + def factory(self, loop, coro, context=None): + return asyncio.tasks._CTask(coro, loop=loop, context=context) + + +class TestEagerPyFreeThreading(TestPyFreeThreading): + def factory(self, loop, coro, context=None): + return asyncio.tasks._PyTask(coro, loop=loop, context=context, eager_start=True) + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestEagerCFreeThreading(TestCFreeThreading, TestCase): + def factory(self, loop, coro, context=None): + return asyncio.tasks._CTask(coro, loop=loop, context=context, eager_start=True) From 4381ae26b959ef355e4020fecbd6e47019f4cabe Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 09:44:26 +0000 Subject: [PATCH 04/10] fix refcounting --- Modules/_asynciomodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 08608c0508137a..05b789b608c0c7 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3748,6 +3748,7 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(loop); return NULL; } + Py_DECREF(eager_tasks); PyObject *eager_iter = PyObject_GetIter(eager_tasks); if (eager_iter == NULL) { Py_DECREF(tasks); From b97212152e799bc9bcbc1605d91d6da755a68583 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 10:08:18 +0000 Subject: [PATCH 05/10] fix refcounting --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 05b789b608c0c7..e3c1c0213439c2 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3748,13 +3748,13 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(loop); return NULL; } - Py_DECREF(eager_tasks); PyObject *eager_iter = PyObject_GetIter(eager_tasks); if (eager_iter == NULL) { Py_DECREF(tasks); Py_DECREF(loop); return NULL; } + Py_DECREF(eager_tasks); PyObject *item; while ((item = PyIter_Next(eager_iter)) != NULL) { if (add_one_task(state, tasks, item, loop) < 0) { From eeb0273aab55308966dddc4aa162fd1731413d45 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 10:25:21 +0000 Subject: [PATCH 06/10] add asserts --- Modules/_asynciomodule.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e3c1c0213439c2..a447fb0d246eb4 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -375,6 +375,8 @@ future_ensure_alive(FutureObj *fut) static int future_schedule_callbacks(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (fut->fut_callback0 != NULL) { /* There's a 1st callback */ @@ -492,10 +494,11 @@ future_init(FutureObj *fut, PyObject *loop) static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (future_ensure_alive(fut)) { return NULL; } - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); if (fut->fut_state != STATE_PENDING) { PyErr_SetString(state->asyncio_InvalidStateError, "invalid state"); @@ -515,6 +518,8 @@ future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) static PyObject * future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc_val = NULL; if (fut->fut_state != STATE_PENDING) { @@ -581,6 +586,8 @@ future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) static PyObject * create_cancelled_error(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc; if (fut->fut_cancelled_exc != NULL) { /* transfer ownership */ @@ -600,6 +607,8 @@ create_cancelled_error(asyncio_state *state, FutureObj *fut) static void future_set_cancelled_error(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc = create_cancelled_error(state, fut); if (exc == NULL) { return; @@ -611,6 +620,8 @@ future_set_cancelled_error(asyncio_state *state, FutureObj *fut) static int future_get_result(asyncio_state *state, FutureObj *fut, PyObject **result) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (fut->fut_state == STATE_CANCELLED) { future_set_cancelled_error(state, fut); return -1; @@ -644,6 +655,8 @@ static PyObject * future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, PyObject *ctx) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (!future_is_alive(fut)) { PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); return NULL; @@ -718,6 +731,8 @@ future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, static PyObject * future_cancel(asyncio_state *state, FutureObj *fut, PyObject *msg) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + fut->fut_log_tb = 0; if (fut->fut_state != STATE_PENDING) { From fd1b9cd16be8f9505dd509bac54a5f05f1ab3d77 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 21 Dec 2024 11:35:26 +0000 Subject: [PATCH 07/10] more asserts --- Modules/_asynciomodule.c | 49 ++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a447fb0d246eb4..ba9d1ed198d8f0 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1666,7 +1666,7 @@ FutureIter_dealloc(futureiterobject *it) } static PySendResult -FutureIter_am_send(futureiterobject *it, +FutureIter_am_send_lock_held(futureiterobject *it, PyObject *Py_UNUSED(arg), PyObject **result) { @@ -1703,6 +1703,19 @@ FutureIter_am_send(futureiterobject *it, return PYGEN_ERROR; } +static PySendResult +FutureIter_am_send(futureiterobject *it, + PyObject *Py_UNUSED(arg), + PyObject **result) +{ + PySendResult res; + Py_BEGIN_CRITICAL_SECTION2(it, it->future); + res = FutureIter_am_send_lock_held(it, Py_None, result); + Py_END_CRITICAL_SECTION2(); + return res; +} + + static PyObject * FutureIter_iternext(futureiterobject *it) { @@ -1923,7 +1936,11 @@ TaskStepMethWrapper_call(TaskStepMethWrapper *o, return NULL; } asyncio_state *state = get_asyncio_state_by_def((PyObject *)o); - return task_step(state, o->sw_task, o->sw_arg); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o->sw_task); + res = task_step(state, o->sw_task, o->sw_arg); + Py_END_CRITICAL_SECTION(); + return res; } static int @@ -2909,6 +2926,8 @@ gen_status_from_result(PyObject **result) static PyObject * task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + int clear_exc = 0; PyObject *result = NULL; PyObject *coro; @@ -3068,8 +3087,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (wrapper == NULL) { goto fail; } + Py_BEGIN_CRITICAL_SECTION(result); tmp = future_add_done_callback(state, (FutureObj*)result, wrapper, task->task_context); + Py_END_CRITICAL_SECTION(); Py_DECREF(wrapper); if (tmp == NULL) { goto fail; @@ -3270,9 +3291,9 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc) if (enter_task(state, task->task_loop, (PyObject*)task) < 0) { return NULL; } - Py_BEGIN_CRITICAL_SECTION(task); + res = task_step_impl(state, task, exc); - Py_END_CRITICAL_SECTION(); + if (res == NULL) { PyObject *exc = PyErr_GetRaisedException(); leave_task(state, task->task_loop, (PyObject*)task); @@ -3351,16 +3372,20 @@ task_eager_start(asyncio_state *state, TaskObj *task) } static PyObject * -task_wakeup(TaskObj *task, PyObject *o) +task_wakeup_lock_held(TaskObj *task, PyObject *o) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + PyObject *result; assert(o); asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); if (Future_CheckExact(state, o) || Task_CheckExact(state, o)) { PyObject *fut_result = NULL; - int res = future_get_result(state, (FutureObj*)o, &fut_result); - + int res; + Py_BEGIN_CRITICAL_SECTION(o); + res = future_get_result(state, (FutureObj*)o, &fut_result); + Py_END_CRITICAL_SECTION(); switch(res) { case -1: assert(fut_result == NULL); @@ -3394,6 +3419,16 @@ task_wakeup(TaskObj *task, PyObject *o) return result; } +static PyObject * +task_wakeup(TaskObj *task, PyObject *o) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(task); + res = task_wakeup_lock_held(task, o); + Py_END_CRITICAL_SECTION(); + return res; +} + /*********************** Functions **************************/ From b62846305cd4e915325c1a70cb9c3082e58628db Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Mon, 23 Dec 2024 08:28:36 +0000 Subject: [PATCH 08/10] more asserts --- Modules/_asynciomodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index ba9d1ed198d8f0..0eacdfc486a5a6 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3057,6 +3057,8 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) static PyObject * task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + int res; PyObject *o; From 86448af5e7df6e469929e7b575a8f19e960ad3e8 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Mon, 23 Dec 2024 08:35:39 +0000 Subject: [PATCH 09/10] fix test --- Lib/test/test_asyncio/test_free_threading.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index e4c28d33e8b280..bba3bdd9552628 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -14,19 +14,19 @@ async def main(): loop = asyncio.get_running_loop() future = loop.create_future() - async def task(): + async def coro(): await future tasks = set() async with asyncio.TaskGroup() as tg: for _ in range(100): - tasks.add(tg.create_task(task())) + tasks.add(tg.create_task(coro())) all_tasks = self.all_tasks(loop) self.assertEqual(len(all_tasks), 101) - for task in tasks: + for task in all_tasks: self.assertEqual(task.get_loop(), loop) self.assertFalse(task.done()) From 0e4a7755ba21819c63664e258cd228fe23c2f245 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 25 Dec 2024 14:44:05 +0000 Subject: [PATCH 10/10] fix crash by using mutex instead of critical_section --- Modules/_asynciomodule.c | 5 +++-- Tools/tsan/suppressions_free_threading.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index b191ab6d6f116f..2c6e52b601314a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -77,8 +77,8 @@ typedef struct { #define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) #ifdef Py_GIL_DISABLED -# define ASYNCIO_STATE_LOCK(state) Py_BEGIN_CRITICAL_SECTION_MUT(&state->mutex) -# define ASYNCIO_STATE_UNLOCK(state) Py_END_CRITICAL_SECTION() +# define ASYNCIO_STATE_LOCK(state) PyMutex_Lock(&state->mutex) +# define ASYNCIO_STATE_UNLOCK(state) PyMutex_Unlock(&state->mutex) #else # define ASYNCIO_STATE_LOCK(state) ((void)state) # define ASYNCIO_STATE_UNLOCK(state) ((void)state) @@ -3798,6 +3798,7 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) int err = 0; ASYNCIO_STATE_LOCK(state); struct llist_node *node; + llist_for_each_safe(node, &state->asyncio_tasks_head) { TaskObj *task = llist_data(node, TaskObj, task_node); Py_INCREF(task); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index e5eb665ae212de..2d871dd397220d 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -43,6 +43,7 @@ race_top:write_thread_id race_top:PyThreadState_Clear # Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c race_top:set_default_allocator_unlocked +race_top:socket_socketpair # https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 thread:pthread_create