docs: add troubleshoot for missing metrics for Argo installation (#3493)

Signed-off-by: Dominik Rosiek <[email protected]>

docs/troubleshoot-collection.md

@@ -21,6 +21,7 @@
     - [Check the `/metrics` endpoint for Kubernetes services](#check-the-metrics-endpoint-for-kubernetes-services)
   - [Check the Prometheus UI](#check-the-prometheus-ui)
   - [Check Prometheus Remote Storage](#check-prometheus-remote-storage)
+  - [Missing metrics for ArgoCD installation](#missing-metrics-for-argocd-installation)
 - [Common Issues](#common-issues)
   - [Missing metrics - cannot see cluster in Explore](#missing-metrics---cannot-see-cluster-in-explore)
   - [Pod stuck in `ContainerCreating` state](#pod-stuck-in-containercreating-state)
@@ -340,6 +341,34 @@ You [check Prometheus logs](#prometheus-logs) to verify there are no errors duri
 
 You can also check `prometheus_remote_storage_.*` metrics to look for success/failure attempts.
 
+### Missing metrics for ArgoCD installation
+
+There is known issue with Argo CD and metrics collection. If you override `spec.source.helm.releaseName` in the `Application` or
+`ApplicationSet`, which are used to configure your application in Argo CD, then Kube State and Node metrics are not collected due to the
+following:
+
+Service Monitor is looking for service labeled with `app.kubernetes.io/instance: <spec.source.helm.releaseName>`, but the label is actually
+`app.kubernetes.io/instance: <metadata.name>`.
+
+In order to fix it, you need to ensure that the labels are matching and you can do it by adding the following to `user-values.yaml`:
+
+```
+kube-prometheus-stack:
+  kube-state-metrics:
+    prometheus:
+      monitor:
+        selectorOverride:
+          app.kubernetes.io/instance: <metadata.name>
+          app.kubernetes.io/name: kube-state-metrics
+  prometheus-node-exporter:
+    prometheus:
+      monitor:
+        selectorOverride:
+          app.kubernetes.io/name: prometheus-node-exporter
+```
+
+where `metadata.name` is value from Argo Application manifest
+
 ## Common Issues
 
 ### Missing metrics - cannot see cluster in Explore

vagrant/Makefile

@@ -299,3 +299,19 @@ istio-disable:
 
 restart-pods:
 	kubectl -n sumologic delete pod --all --force --grace-period=0
+
+install-argocd-binary:
+	curl -LO https://github.com/argoproj/argo-cd/releases/download/v2.9.3/argocd-linux-amd64
+	chmod +x argocd-linux-amd64
+	sudo mv argocd-linux-amd64 /usr/local/bin/argocd
+
+install-argocd:
+	${mkfile_path} \
+		apply-receiver-mock
+	kubectl create namespace argocd
+	kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
+	kubectl patch svc argocd-server -n argocd -p '{"spec": {"type": "LoadBalancer"}}'
+	argocd admin initial-password -n argocd
+
+expose-argocd:
+	kubectl port-forward svc/argocd-server -n argocd 8080:443 --address 0.0.0.0

vagrant/k8s/argo.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: collection
+  namespace: argocd
+spec:
+  destination:
+    namespace: sumologic
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    chart: sumologic
+    helm:
+      releaseName: collection2
+      values:
+        '{"sumologic": {"accessId": "dummy", "accessKey": "dummy", "endpoint": "http://receiver-mock.receiver-mock:3000/terraform/api/"}}'
+    repoURL: https://sumologic.github.io/sumologic-kubernetes-collection
+    targetRevision: 4.3.1
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true