From 30e6d398c893eef6a917f9f38d39addd760c14a6 Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Sun, 10 Nov 2024 19:04:29 +0100 Subject: [PATCH 1/2] Fix crash at shutdown --- .../functions_to_wrap.c | 28 +++++++++++++++++-- .../CorProfilerCallback.cpp | 7 +++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c index 84417cbda8f9..876392d37e84 100644 --- a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c +++ b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c @@ -61,10 +61,13 @@ __thread unsigned long long functions_entered_counter = 0; __attribute__((visibility("hidden"))) atomic_int is_app_crashing = 0; +__attribute__((visibility("hidden"))) +__thread int8_t is_thread_routine_ended = 0; + // this function is called by the profiler unsigned long long dd_inside_wrapped_functions() { - return functions_entered_counter + is_app_crashing; + return functions_entered_counter + is_app_crashing + is_thread_routine_ended; } #if defined(__aarch64__) @@ -541,6 +544,23 @@ int execve(const char* pathname, char* const argv[], char* const envp[]) #ifdef DD_ALPINE +struct pthread_wrapped_arg +{ + void* (*func)(void*); + void* orig_arg; +}; + +__attribute__((visibility("hidden"))) +static void* entry2(void* arg) +{ + struct pthread_wrapped_arg* new_arg = (struct pthread_wrapped_arg*)arg; + void* result = new_arg->func(new_arg->orig_arg); + // check if we should call the profiler + is_thread_routine_ended = 1; + free(new_arg); + return result; +} + /* Function pointers to hold the value of the glibc functions */ static int (*__real_pthread_create)(pthread_t* restrict res, const pthread_attr_t* restrict attrp, void* (*entry)(void*), void* restrict arg) = NULL; @@ -551,7 +571,11 @@ int pthread_create(pthread_t* restrict res, const pthread_attr_t* restrict attrp ((char*)&functions_entered_counter)[ENTERED_PTHREAD_CREATE]++; // call the real pthread_create (libc/musl-libc) - int result = __real_pthread_create(res, attrp, entry, arg); + struct pthread_wrapped_arg* new_arg = (struct pthread_wrapped_arg*)malloc(sizeof(struct pthread_wrapped_arg)); + new_arg->func = entry; + new_arg->orig_arg = arg; + + int result = __real_pthread_create(res, attrp, entry2, new_arg); ((char*)&functions_entered_counter)[ENTERED_PTHREAD_CREATE]--; diff --git a/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp b/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp index b1801c13e6c1..87ba5bf8c316 100644 --- a/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp +++ b/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp @@ -1320,6 +1320,13 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::Shutdown() { Log::Info("CorProfilerCallback::Shutdown()"); +#ifdef LINUX + if (_pCpuProfiler != nullptr) + { + _pCpuProfiler->Stop(); + } +#endif + // A final .pprof should be generated before exiting // The aggregator must be stopped before the provider, since it will call them to get the last samples _pStackSamplerLoopManager->Stop(); From e0b4bb8ae208bcb70b25c63893951b9d00b2d024 Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Tue, 12 Nov 2024 11:33:32 +0100 Subject: [PATCH 2/2] Add a callback into the profiler --- .../Datadog.Linux.ApiWrapper/common.c | 12 +++++ .../Datadog.Linux.ApiWrapper/common.h | 1 + .../functions_to_wrap.c | 10 ++--- .../CorProfilerCallback.cpp | 44 +++++++++++++++++++ .../CorProfilerCallback.h | 1 + 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.c b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.c index 1ab8309b0957..dd072db2a959 100644 --- a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.c +++ b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.c @@ -36,6 +36,18 @@ __attribute__((visibility("hidden"))) inline int is_interrupted_by_profiler(int return rc == -1L && error_code == EINTR && interrupted_by_profiler != 0; } + +void (*volatile dd_on_thread_routine_finished)() = NULL; +__attribute__((visibility("hidden"))) inline void __dd_on_thread_routine_finished() +{ + void (*volatile on_thread_routine_finished)() = dd_on_thread_routine_finished; + + if (on_thread_routine_finished == NULL) + return; + + on_thread_routine_finished(); +} + char* dlerror(void) __attribute__((weak)); static void* s_libdl_handle = NULL; static void* s_libpthread_handle = NULL; diff --git a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.h b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.h index 7ced49a0c0a1..3108d1ec128c 100644 --- a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.h +++ b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.h @@ -36,5 +36,6 @@ extern int (*volatile dd_set_shared_memory)(volatile int*); int is_interrupted_by_profiler(int rc, int error_code, int interrupted_by_profiler); int __dd_set_shared_memory(volatile int* mem); void __dd_notify_libraries_cache_update(); +void __dd_on_thread_routine_finished(); void *__dd_dlsym(void *handle, const char *symbol); \ No newline at end of file diff --git a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c index 876392d37e84..0108340b8e6d 100644 --- a/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c +++ b/profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c @@ -61,13 +61,10 @@ __thread unsigned long long functions_entered_counter = 0; __attribute__((visibility("hidden"))) atomic_int is_app_crashing = 0; -__attribute__((visibility("hidden"))) -__thread int8_t is_thread_routine_ended = 0; - // this function is called by the profiler unsigned long long dd_inside_wrapped_functions() { - return functions_entered_counter + is_app_crashing + is_thread_routine_ended; + return functions_entered_counter + is_app_crashing; } #if defined(__aarch64__) @@ -555,9 +552,10 @@ static void* entry2(void* arg) { struct pthread_wrapped_arg* new_arg = (struct pthread_wrapped_arg*)arg; void* result = new_arg->func(new_arg->orig_arg); - // check if we should call the profiler - is_thread_routine_ended = 1; free(new_arg); + // Call into the profiler to do extra cleanup. + // This is *useful* at shutdown time to avoid crashing. + __dd_on_thread_routine_finished(); return result; } diff --git a/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp b/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp index 87ba5bf8c316..9e1d3dbd5688 100644 --- a/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp +++ b/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp @@ -82,6 +82,10 @@ extern "C" __attribute__((visibility("default"))) const char* Profiler_Version = // Initialization CorProfilerCallback* CorProfilerCallback::_this = nullptr; +#ifdef LINUX +extern "C" void (*volatile dd_on_thread_routine_finished)() __attribute__((weak)); +#endif + CorProfilerCallback::CorProfilerCallback(std::shared_ptr pConfiguration) : _pConfiguration{std::move(pConfiguration)} { @@ -94,6 +98,12 @@ CorProfilerCallback::CorProfilerCallback(std::shared_ptr pConfig #ifndef _WINDOWS CGroup::Initialize(); #endif +#if defined(LINUX) + if (&dd_on_thread_routine_finished != nullptr) + { + dd_on_thread_routine_finished = CorProfilerCallback::OnThreadRoutineFinished; + } +#endif } // Cleanup @@ -106,6 +116,13 @@ CorProfilerCallback::~CorProfilerCallback() #ifndef _WINDOWS CGroup::Cleanup(); #endif + +#if defined(LINUX) + if (&dd_on_thread_routine_finished != nullptr) + { + dd_on_thread_routine_finished = nullptr; + } +#endif } void CorProfilerCallback::InitializeServices() @@ -1552,6 +1569,31 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::ThreadCreated(ThreadID threadId) return S_OK; } +#ifdef LINUX +void CorProfilerCallback::OnThreadRoutineFinished() +{ + auto threadInfo = ManagedThreadInfo::CurrentThreadInfo; + if (threadInfo == nullptr) + { + return; + } + + auto myThis = _this; + if (myThis == nullptr) + { + return; + } + + auto* cpuProfiler = myThis->_pCpuProfiler; + if (cpuProfiler == nullptr) + { + return; + } + + cpuProfiler->UnregisterThread(threadInfo); +} +#endif + HRESULT STDMETHODCALLTYPE CorProfilerCallback::ThreadDestroyed(ThreadID threadId) { Log::Debug("Callback invoked: ThreadDestroyed(threadId=0x", std::hex, threadId, std::dec, ")"); @@ -1562,6 +1604,8 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::ThreadDestroyed(ThreadID threadId return S_OK; } + ManagedThreadInfo::CurrentThreadInfo = nullptr; + std::shared_ptr pThreadInfo; Log::Debug("Removing thread ", std::hex, threadId, " from the trace context threads list."); if (_pCodeHotspotsThreadList->UnregisterThread(threadId, pThreadInfo)) diff --git a/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.h b/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.h index 355d4be771d9..a1be92e5733e 100644 --- a/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.h +++ b/profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.h @@ -283,6 +283,7 @@ private : static void InspectProcessorInfo(); static const char* SysInfoProcessorArchitectureToStr(WORD wProcArch); static void PrintEnvironmentVariables(); + static void OnThreadRoutineFinished(); void InspectRuntimeVersion(ICorProfilerInfo5* pCorProfilerInfo, USHORT& major, USHORT& minor, COR_PRF_RUNTIME_TYPE& runtimeType); void DisposeInternal();