From 8eb0ff6c3c7bc24610084665c47925c4cf740e8d Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Thu, 28 Dec 2023 13:28:58 -0500 Subject: [PATCH] Remove error message from low mem limits --- gpu-operator/troubleshooting.rst | 50 -------------------------------- 1 file changed, 50 deletions(-) diff --git a/gpu-operator/troubleshooting.rst b/gpu-operator/troubleshooting.rst index 975a7d2ca..8993b70da 100644 --- a/gpu-operator/troubleshooting.rst +++ b/gpu-operator/troubleshooting.rst @@ -64,56 +64,6 @@ can get stuck in a crash loop. PIDPressure False Tue, 26 Dec 2023 14:01:31 +0000 Tue, 12 Dec 2023 19:47:47 +0000 KubeletHasSufficientPID kubelet has sufficient PID available Ready True Tue, 26 Dec 2023 14:01:31 +0000 Thu, 14 Dec 2023 19:15:13 +0000 KubeletReady kubelet is posting ready status -- The logs from the pod include a fatal error: - - .. code-block:: console - - $ kubectl logs -n gpu-operator -l app=gpu-operator - - *Partial Output* - - .. code-block:: output - :emphasize-lines: 1 - - fatal error: concurrent map read and map write - - goroutine 676 [running]: - k8s.io/apimachinery/pkg/runtime.(*Scheme).ObjectKinds(0xc0001fc000, {0x1ea20f0?, 0xc0008b4770}) - /workspace/vendor/k8s.io/apimachinery/pkg/runtime/scheme.go:264 +0xce - sigs.k8s.io/controller-runtime/pkg/client/apiutil.GVKForObject({0x1ea20f0?, 0xc0008b4770}, 0xc00133d4e0?) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/client/apiutil/apimachinery.go:98 +0x245 - sigs.k8s.io/controller-runtime/pkg/cache.(*informerCache).objectTypeForListObject(0xc0000123c0, {0x1ebe020?, 0xc0008b4770}) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/cache/informer_cache.go:94 +0x87 - sigs.k8s.io/controller-runtime/pkg/cache.(*informerCache).List(0xc0000123c0, {0x1eb5ca8, 0xc000618cd0}, {0x1ebe020, 0xc0008b4770}, {0x2c7cf70, 0x0, 0x0}) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/cache/informer_cache.go:73 +0x71 - sigs.k8s.io/controller-runtime/pkg/client.(*delegatingReader).List(0xc000c4b480, {0x1eb5ca8, 0xc000618cd0}, {0x1ebe020?, 0xc0008b4770?}, {0x2c7cf70, 0x0, 0x0}) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/client/split.go:140 +0x114 - github.com/NVIDIA/gpu-operator/controllers.addWatchNewGPUNode.func1({0x199a6a0?, 0xc002873b30?}) - /workspace/controllers/clusterpolicy_controller.go:228 +0x9a - sigs.k8s.io/controller-runtime/pkg/handler.(*enqueueRequestsFromMapFunc).mapAndEnqueue(0x44?, {0x1ebf938, 0xc000158660}, {0x1ecc6c0?, 0xc001c04fc0?}, 0xa8?) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/handler/enqueue_mapped.go:80 +0x46 - sigs.k8s.io/controller-runtime/pkg/handler.(*enqueueRequestsFromMapFunc).Create(0xc000095900?, {{0x1ecc6c0?, 0xc001c04fc0?}}, {0x1ebf938, 0xc000158660}) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/handler/enqueue_mapped.go:57 +0xd2 - sigs.k8s.io/controller-runtime/pkg/source/internal.EventHandler.OnAdd({{0x1eb66b8, 0xc000012498}, {0x1ebf938, 0xc000158660}, {0xc000b891f0, 0x1, 0x1}}, {0x1bea560?, 0xc001c04fc0}) - /workspace/vendor/sigs.k8s.io/controller-runtime/pkg/source/internal/eventsource.go:63 +0x295 - k8s.io/client-go/tools/cache.(*processorListener).run.func1() - /workspace/vendor/k8s.io/client-go/tools/cache/shared_informer.go:818 +0x134 - k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?) - /workspace/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:157 +0x3e - k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0006fc738?, {0x1e9eae0, 0xc0014c48a0}, 0x1, 0xc000b1a540) - /workspace/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:158 +0xb6 - k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x1ebcb18?, 0x3b9aca00, 0x0, 0x51?, 0xc0006fc7b0?) - /workspace/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:135 +0x89 - k8s.io/apimachinery/pkg/util/wait.Until(...) - /workspace/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:92 - k8s.io/client-go/tools/cache.(*processorListener).run(0xc000766f80) - /workspace/vendor/k8s.io/client-go/tools/cache/shared_informer.go:812 +0x6b - k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() - /workspace/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:75 +0x5a - created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start - /workspace/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x85 - - .. rubric:: Root Cause :class: h4