first set of metrics

Makefile

@@ -163,6 +163,12 @@ GOVULNCHECK_VER := v1.0.0
 GOVULNCHECK := $(abspath $(TOOLS_BIN_DIR)/$(GOVULNCHECK_BIN)-$(GOVULNCHECK_VER))
 GOVULNCHECK_PKG := golang.org/x/vuln/cmd/govulncheck
 
+KUBE_STATE_METRICS_VER := e31ed9ab
+KUBE_STATE_METRICS_BIN := kube-state-metrics
+KUBE_STATE_METRICS := $(abspath $(TOOLS_BIN_DIR)/$(KUBE_STATE_METRICS_BIN)-$(KUBE_STATE_METRICS_VER))
+KUBE_STATE_METRICS_PKG := k8s.io/kube-state-metrics/v2
+KUBE_STATE_METRICS_MOD_REPLACE := $(KUBE_STATE_METRICS_PKG)=github.com/chrischdi/kube-state-metrics/v2@$(KUBE_STATE_METRICS_VER)
+
 GOVC_VER := $(shell cat go.mod | grep "github.com/vmware/govmomi" | awk '{print $$NF}')
 GOVC_BIN := govc
 GOVC := $(abspath $(TOOLS_BIN_DIR)/$(GOVC_BIN)-$(GOVC_VER))
@@ -184,6 +190,8 @@ RELEASE_NOTES_VER := $(CAPI_HACK_TOOLS_VER)
 RELEASE_NOTES_BIN := release-notes
 RELEASE_NOTES := $(abspath $(TOOLS_BIN_DIR)/$(RELEASE_NOTES_BIN)-$(RELEASE_NOTES_VER))
 RELEASE_NOTES_PKG := sigs.k8s.io/cluster-api/hack/tools/release
+# let go resolve the tools version via the CAPI version.
+RELEASE_NOTES_MOD_REQUIRE := sigs.k8s.io/cluster-api/hack/tools@$(RELEASE_NOTES_VER)
 
 # Define Docker related variables. Releases should modify and double check these vars.
 REGISTRY ?= gcr.io/$(shell gcloud config get-value project)
@@ -237,7 +245,7 @@ help:  # Display this help
 
 .PHONY: generate
 generate: ## Run all generate targets
-	$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-flavors
+	$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-flavors generate-metrics-config
 
 .PHONY: generate-manifests
 generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
@@ -316,6 +324,9 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	# for DHCP overrides
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/dhcp-overrides > $(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml
 
+.PHONY: generate-metrics-config
+generate-metrics-config: $(KUBE_STATE_METRICS) ## Generate ./crd-metrics-config.yaml
+	$(KUBE_STATE_METRICS) generate ./apis/v1beta1/... > config/metrics/crd-metrics-config.yaml
 
 ## --------------------------------------
 ## Lint / Verify
@@ -407,7 +418,6 @@ verify-flavors: $(FLAVOR_DIR) generate-flavors ## Verify generated flavors
 		echo "flavor files in templates directory are out of date"; exit 1; \
 	fi
 
-
 ## --------------------------------------
 ## Build
 ## --------------------------------------
@@ -606,7 +616,6 @@ e2e-flavors-main: $(RELEASE_DIR)
 	mkdir -p $(RELEASE_DIR)/main
 	$(MAKE) generate-flavors FLAVOR_DIR=$(RELEASE_DIR)/main
 
-
 .PHONY: generate-flavors
 generate-flavors: $(FLAVOR_DIR)
 	go run ./packaging/flavorgen --output-dir $(FLAVOR_DIR)
@@ -763,13 +772,15 @@ $(GOLANGCI_LINT_BIN): $(GOLANGCI_LINT) ## Build a local copy of golangci-lint.
 .PHONY: $(GOVULNCHECK_BIN)
 $(GOVULNCHECK_BIN): $(GOVULNCHECK) ## Build a local copy of govulncheck.
 
+.PHONY: $(KUBE_STATE_METRICS_BIN)
+$(KUBE_STATE_METRICS_BIN): $(KUBE_STATE_METRICS) ## Build a local copy of metric-gen.
+
 .PHONY: $(GOVC_BIN)
 $(GOVC_BIN): $(GOVC) ## Build a local copy of govc.
 
 .PHONY: $(KIND_BIN)
 $(KIND_BIN): $(KIND) ## Build a local copy of kind.
 
-
 .PHONY: $(RELEASE_NOTES_BIN)
 $(RELEASE_NOTES_BIN): $(RELEASE_NOTES) ## Build a local copy of release-notes.
 
@@ -815,15 +826,17 @@ $(GOLANGCI_LINT): # Build golangci-lint.
 $(GOVULNCHECK): # Build govulncheck.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOVULNCHECK_PKG) $(GOVULNCHECK_BIN) $(GOVULNCHECK_VER)
 
+$(KUBE_STATE_METRICS): # Build kube-state-metrics.
+	GOBIN=$(TOOLS_BIN_DIR) GOMOD_REPLACE="$(KUBE_STATE_METRICS_MOD_REPLACE)" $(GO_TOOLS_BUILD) $(KUBE_STATE_METRICS_PKG) $(KUBE_STATE_METRICS_BIN) $(KUBE_STATE_METRICS_VER)
+
 $(GOVC): # Build GOVC.
 	CGO_ENABLED=0 GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOVC_PKG) $(GOVC_BIN) $(GOVC_VER)
 
 $(KIND): # Build kind.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KIND_PKG) $(KIND_BIN) $(KIND_VER)
 
-
 $(RELEASE_NOTES): # Build release-notes.
-	GOBIN=$(TOOLS_BIN_DIR) $(GO_TOOLS_BUILD) $(RELEASE_NOTES_PKG) $(RELEASE_NOTES_BIN) $(RELEASE_NOTES_VER)
+	GOBIN=$(TOOLS_BIN_DIR) GOMOD_REQUIRE="$(RELEASE_NOTES_MOD_REQUIRE)" $(GO_TOOLS_BUILD) $(RELEASE_NOTES_PKG) $(RELEASE_NOTES_BIN) $(RELEASE_NOTES_VER)
 
 ## --------------------------------------
 ## Helpers

apis/v1beta1/vspherecluster_types.go

@@ -91,6 +91,8 @@ type VSphereClusterStatus struct {
 
 	// Conditions defines current service state of the VSphereCluster.
 	// +optional
+	// +Metrics:stateset:name="status_condition",help="The condition of a vspherecluster.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
+	// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspherecluster.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
 	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 
 	// FailureDomains is a list of failure domain objects synced from the infrastructure provider.
@@ -110,6 +112,15 @@ type VSphereClusterStatus struct {
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"
 
 // VSphereCluster is the Schema for the vsphereclusters API
+// +Metrics:gvk:namePrefix="capi_vspherecluster"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
+// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspherecluster is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+// +Metrics:info:name="info",help="Information about a vspherecluster.",labelsFromPath={status_vsphere_version:.status.vCenterVersion,spec_server:.spec.server,identity_reference_kind:".spec.identityRef.kind",identity_reference_name:".spec.identityRef.name"}
 type VSphereCluster struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

apis/v1beta1/vsphereclusteridentity_types.go

@@ -45,6 +45,8 @@ type VSphereClusterIdentityStatus struct {
 
 	// Conditions defines current service state of the VSphereCluster.
 	// +optional
+	// +Metrics:stateset:name="status_condition",help="The condition of a vsphereclusteridentity.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
+	// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vsphereclusteridentity.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
 	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 }
 
@@ -85,6 +87,15 @@ func (c *VSphereClusterIdentity) SetConditions(conditions clusterv1.Conditions)
 // +kubebuilder:subresource:status
 
 // VSphereClusterIdentity defines the account to be used for reconciling clusters
+// +Metrics:gvk:namePrefix="capi_vsphereclusteridentity"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
+// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vsphereclusteridentity is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+// +Metrics:info:name="info",help="Information about a vsphereclusteridentity.",labelsFromPath={name:.metadata.name}
 type VSphereClusterIdentity struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

apis/v1beta1/vspheredeploymentzone_types.go

@@ -81,6 +81,8 @@ type VSphereDeploymentZoneStatus struct {
 
 	// Conditions defines current service state of the VSphereMachine.
 	// +optional
+	// +Metrics:stateset:name="status_condition",help="The condition of a vspheredeploymentzone.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
+	// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspheredeploymentzone.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
 	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 }
 
@@ -90,6 +92,15 @@ type VSphereDeploymentZoneStatus struct {
 // +kubebuilder:subresource:status
 
 // VSphereDeploymentZone is the Schema for the vspheredeploymentzones API
+// +Metrics:gvk:namePrefix="capi_vspheredeploymentzone"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
+// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspheredeploymentzone is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+// +Metrics:info:name="info",help="Information about a vspheredeploymentzone.",labelsFromPath={name:.metadata.name}
 type VSphereDeploymentZone struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

apis/v1beta1/vspherefailuredomain_types.go

@@ -95,6 +95,15 @@ type FailureDomainHosts struct {
 // +kubebuilder:resource:path=vspherefailuredomains,scope=Cluster,categories=cluster-api
 
 // VSphereFailureDomain is the Schema for the vspherefailuredomains API
+// +Metrics:gvk:namePrefix="capi_vspherefailuredomain"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
+// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspherefailuredomain is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+// +Metrics:info:name="info",help="Information about a vspherefailuredomain.",labelsFromPath={name:.metadata.name}
 type VSphereFailureDomain struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

apis/v1beta1/vspheremachine_types.go

@@ -78,6 +78,7 @@ type VSphereMachineStatus struct {
 	Ready bool `json:"ready"`
 
 	// Addresses contains the VSphere instance associated addresses.
+	// +Metrics:info:name="status_addresses",help="Address information about a vspheremachine.",labelsFromPath={address:".address",type:".type"}
 	Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"`
 
 	// Network returns the network status for each of the machine's configured
@@ -125,6 +126,8 @@ type VSphereMachineStatus struct {
 
 	// Conditions defines current service state of the VSphereMachine.
 	// +optional
+	// +Metrics:stateset:name="status_condition",help="The condition of a vspheremachine.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
+	// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspheremachine.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
 	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 }
 
@@ -139,6 +142,15 @@ type VSphereMachineStatus struct {
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"
 
 // VSphereMachine is the Schema for the vspheremachines API
+// +Metrics:gvk:namePrefix="capi_vspheremachine"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
+// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspheremachine is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+// +Metrics:info:name="info",help="Information about a vspheremachine.",labelsFromPath={provider_id:.spec.providerID}
 type VSphereMachine struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

apis/v1beta1/vspherevm_types.go

@@ -104,6 +104,7 @@ type VSphereVMStatus struct {
 	// This field is required at runtime for other controllers that read
 	// this CRD as unstructured data.
 	// +optional
+	// +Metrics:info:name="status_addresses",help="Address information about a vspherevm.",labelsFromPath={address:"."}
 	Addresses []string `json:"addresses,omitempty"`
 
 	// CloneMode is the type of clone operation used to clone this VM. Since
@@ -165,6 +166,8 @@ type VSphereVMStatus struct {
 
 	// Conditions defines current service state of the VSphereVM.
 	// +optional
+	// +Metrics:stateset:name="status_condition",help="The condition of a vspherevm.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
+	// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspherevm.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
 	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 
 	// ModuleUUID is the unique identifier for the vCenter cluster module construct
@@ -187,6 +190,15 @@ type VSphereVMStatus struct {
 // +kubebuilder:subresource:status
 
 // VSphereVM is the Schema for the vspherevms API
+// +Metrics:gvk:namePrefix="capi_vspherevm"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
+// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspherevm is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+// +Metrics:info:name="info",help="Information about a vspherevm.",labelsFromPath={status_clonemode:.status.cloneMode,bootstrap_reference_kind:".spec.bootstrapRef.kind",bootstrap_reference_name:".spec.bootstrapRef.name",status_vmref:".status.vmRef"}
 type VSphereVM struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

hack/go-tools-build.sh

@@ -37,7 +37,10 @@ if [ -z "${GOBIN}" ]; then
   exit 1
 fi
 
-CAPI_HACK_TOOLS="sigs.k8s.io/cluster-api/hack/tools"
+export GOWORK="off"
+
+GOMOD_REPLACE=${GOMOD_REPLACE:=}
+GOMOD_REQUIRE=${GOMOD_REQUIRE:=}
 
 rm -f "${GOBIN}/${2}"* || true
 
@@ -52,8 +55,13 @@ cd "${TMP_MODULE_DIR}"
 
 # Initialize a go module and place a tools.go file for building the binary.
 go mod init "tools"
-# Set require for "sigs.k8s.io/cluster-api" to let go resolve the tools version via the CAPI version.
-go mod edit -require "${CAPI_HACK_TOOLS}@${3}"
+
+for PARAM in ${GOMOD_REPLACE}; do
+  eval go mod edit -replace "${PARAM}"
+done
+for PARAM in ${GOMOD_REQUIRE}; do
+  eval go mod edit -require "${PARAM}"
+done
 
 # Create go file which imports the required package and resolve dependencies.
 cat << EOF > tools.go