[Profiler] Fix crash at shutdown with the timer_create-based CPU profiler

profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.c

@@ -36,6 +36,18 @@ __attribute__((visibility("hidden"))) inline int is_interrupted_by_profiler(int
     return rc == -1L && error_code == EINTR && interrupted_by_profiler != 0;
 }
 
+
+void (*volatile dd_on_thread_routine_finished)() = NULL;
+__attribute__((visibility("hidden"))) inline void __dd_on_thread_routine_finished()
+{
+    void (*volatile on_thread_routine_finished)() = dd_on_thread_routine_finished;
+
+    if (on_thread_routine_finished == NULL)
+        return;
+
+    on_thread_routine_finished();
+}
+
 char* dlerror(void) __attribute__((weak));
 static void* s_libdl_handle = NULL;
 static void* s_libpthread_handle = NULL;

profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/common.h

@@ -36,5 +36,6 @@ extern int (*volatile dd_set_shared_memory)(volatile int*);
 int is_interrupted_by_profiler(int rc, int error_code, int interrupted_by_profiler);
 int __dd_set_shared_memory(volatile int* mem);
 void __dd_notify_libraries_cache_update();
+void __dd_on_thread_routine_finished();
 
 void *__dd_dlsym(void *handle, const char *symbol);

profiler/src/ProfilerEngine/Datadog.Linux.ApiWrapper/functions_to_wrap.c

@@ -541,6 +541,24 @@ int execve(const char* pathname, char* const argv[], char* const envp[])
 
 #ifdef DD_ALPINE
 
+struct pthread_wrapped_arg
+{
+    void* (*func)(void*);
+    void* orig_arg;
+};
+
+__attribute__((visibility("hidden")))
+static void* entry2(void* arg)
+{
+    struct pthread_wrapped_arg* new_arg = (struct pthread_wrapped_arg*)arg;
+    void* result = new_arg->func(new_arg->orig_arg);
+    free(new_arg);
+    // Call into the profiler to do extra cleanup.
+    // This is *useful* at shutdown time to avoid crashing.
+    __dd_on_thread_routine_finished();
+    return result;
+}
+
 /* Function pointers to hold the value of the glibc functions */
 static int (*__real_pthread_create)(pthread_t* restrict res, const pthread_attr_t* restrict attrp, void* (*entry)(void*), void* restrict arg) = NULL;
 
@@ -551,7 +569,11 @@ int pthread_create(pthread_t* restrict res, const pthread_attr_t* restrict attrp
     ((char*)&functions_entered_counter)[ENTERED_PTHREAD_CREATE]++;
 
     // call the real pthread_create (libc/musl-libc)
-    int result = __real_pthread_create(res, attrp, entry, arg);
+    struct pthread_wrapped_arg* new_arg = (struct pthread_wrapped_arg*)malloc(sizeof(struct pthread_wrapped_arg));
+    new_arg->func = entry;
+    new_arg->orig_arg = arg;
+
+    int result = __real_pthread_create(res, attrp, entry2, new_arg);
 
     ((char*)&functions_entered_counter)[ENTERED_PTHREAD_CREATE]--;
 

profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp

@@ -82,6 +82,10 @@ extern "C" __attribute__((visibility("default"))) const char* Profiler_Version =
 // Initialization
 CorProfilerCallback* CorProfilerCallback::_this = nullptr;
 
+#ifdef LINUX
+extern "C" void (*volatile dd_on_thread_routine_finished)() __attribute__((weak));
+#endif
+
 CorProfilerCallback::CorProfilerCallback(std::shared_ptr<IConfiguration> pConfiguration) :
     _pConfiguration{std::move(pConfiguration)}
 {
@@ -94,6 +98,12 @@ CorProfilerCallback::CorProfilerCallback(std::shared_ptr<IConfiguration> pConfig
 #ifndef _WINDOWS
     CGroup::Initialize();
 #endif
+#if defined(LINUX)
+    if (&dd_on_thread_routine_finished != nullptr)
+    {
+        dd_on_thread_routine_finished = CorProfilerCallback::OnThreadRoutineFinished;
+    }
+#endif
 }
 
 // Cleanup
@@ -106,6 +116,13 @@ CorProfilerCallback::~CorProfilerCallback()
 #ifndef _WINDOWS
     CGroup::Cleanup();
 #endif
+
+#if defined(LINUX)
+    if (&dd_on_thread_routine_finished != nullptr)
+    {
+        dd_on_thread_routine_finished = nullptr;
+    }
+#endif
 }
 
 void CorProfilerCallback::InitializeServices()
@@ -1320,6 +1337,13 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::Shutdown()
 {
     Log::Info("CorProfilerCallback::Shutdown()");
 
+#ifdef LINUX
+    if (_pCpuProfiler != nullptr)
+    {
+        _pCpuProfiler->Stop();
+    }
+#endif
+
     // A final .pprof should be generated before exiting
     // The aggregator must be stopped before the provider, since it will call them to get the last samples
     _pStackSamplerLoopManager->Stop();
@@ -1545,6 +1569,31 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::ThreadCreated(ThreadID threadId)
     return S_OK;
 }
 
+#ifdef LINUX
+void CorProfilerCallback::OnThreadRoutineFinished()
+{
+    auto threadInfo = ManagedThreadInfo::CurrentThreadInfo;
+    if (threadInfo == nullptr)
+    {
+        return;
+    }
+
+    auto myThis = _this;
+    if (myThis == nullptr)
+    {
+        return;
+    }
+
+    auto* cpuProfiler = myThis->_pCpuProfiler;
+    if (cpuProfiler == nullptr)
+    {
+        return;
+    }
+
+    cpuProfiler->UnregisterThread(threadInfo);
+}
+#endif
+
 HRESULT STDMETHODCALLTYPE CorProfilerCallback::ThreadDestroyed(ThreadID threadId)
 {
     Log::Debug("Callback invoked: ThreadDestroyed(threadId=0x", std::hex, threadId, std::dec, ")");
@@ -1555,6 +1604,8 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::ThreadDestroyed(ThreadID threadId
         return S_OK;
     }
 
+    ManagedThreadInfo::CurrentThreadInfo = nullptr;
+
     std::shared_ptr<ManagedThreadInfo> pThreadInfo;
     Log::Debug("Removing thread ", std::hex, threadId, " from the trace context threads list.");
     if (_pCodeHotspotsThreadList->UnregisterThread(threadId, pThreadInfo))

profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.h

@@ -283,6 +283,7 @@ private :
     static void InspectProcessorInfo();
     static const char* SysInfoProcessorArchitectureToStr(WORD wProcArch);
     static void PrintEnvironmentVariables();
+    static void OnThreadRoutineFinished();
 
     void InspectRuntimeVersion(ICorProfilerInfo5* pCorProfilerInfo, USHORT& major, USHORT& minor, COR_PRF_RUNTIME_TYPE& runtimeType);
     void DisposeInternal();