Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] ✨ Create kube-state-metrics configuration from markers #2310

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,12 @@ GOVULNCHECK_VER := v1.0.0
GOVULNCHECK := $(abspath $(TOOLS_BIN_DIR)/$(GOVULNCHECK_BIN)-$(GOVULNCHECK_VER))
GOVULNCHECK_PKG := golang.org/x/vuln/cmd/govulncheck

KUBE_STATE_METRICS_VER := e31ed9ab
KUBE_STATE_METRICS_BIN := kube-state-metrics
KUBE_STATE_METRICS := $(abspath $(TOOLS_BIN_DIR)/$(KUBE_STATE_METRICS_BIN)-$(KUBE_STATE_METRICS_VER))
KUBE_STATE_METRICS_PKG := k8s.io/kube-state-metrics/v2
KUBE_STATE_METRICS_MOD_REPLACE := $(KUBE_STATE_METRICS_PKG)=github.com/chrischdi/kube-state-metrics/v2@$(KUBE_STATE_METRICS_VER)

GOVC_VER := $(shell cat go.mod | grep "github.com/vmware/govmomi" | awk '{print $$NF}')
GOVC_BIN := govc
GOVC := $(abspath $(TOOLS_BIN_DIR)/$(GOVC_BIN)-$(GOVC_VER))
Expand All @@ -184,6 +190,8 @@ RELEASE_NOTES_VER := $(CAPI_HACK_TOOLS_VER)
RELEASE_NOTES_BIN := release-notes
RELEASE_NOTES := $(abspath $(TOOLS_BIN_DIR)/$(RELEASE_NOTES_BIN)-$(RELEASE_NOTES_VER))
RELEASE_NOTES_PKG := sigs.k8s.io/cluster-api/hack/tools/release
# let go resolve the tools version via the CAPI version.
RELEASE_NOTES_MOD_REQUIRE := sigs.k8s.io/cluster-api/hack/tools@$(RELEASE_NOTES_VER)
sbueringer marked this conversation as resolved.
Show resolved Hide resolved

# Define Docker related variables. Releases should modify and double check these vars.
REGISTRY ?= gcr.io/$(shell gcloud config get-value project)
Expand Down Expand Up @@ -237,7 +245,7 @@ help: # Display this help

.PHONY: generate
generate: ## Run all generate targets
$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-flavors
$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-flavors generate-metrics-config

.PHONY: generate-manifests
generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
Expand Down Expand Up @@ -316,6 +324,9 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
# for DHCP overrides
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/dhcp-overrides > $(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml

.PHONY: generate-metrics-config
generate-metrics-config: $(KUBE_STATE_METRICS) ## Generate ./crd-metrics-config.yaml
$(KUBE_STATE_METRICS) generate ./apis/v1beta1/... > config/metrics/crd-metrics-config.yaml

## --------------------------------------
## Lint / Verify
Expand Down Expand Up @@ -407,7 +418,6 @@ verify-flavors: $(FLAVOR_DIR) generate-flavors ## Verify generated flavors
echo "flavor files in templates directory are out of date"; exit 1; \
fi


## --------------------------------------
## Build
## --------------------------------------
Expand Down Expand Up @@ -606,7 +616,6 @@ e2e-flavors-main: $(RELEASE_DIR)
mkdir -p $(RELEASE_DIR)/main
$(MAKE) generate-flavors FLAVOR_DIR=$(RELEASE_DIR)/main


.PHONY: generate-flavors
generate-flavors: $(FLAVOR_DIR)
go run ./packaging/flavorgen --output-dir $(FLAVOR_DIR)
Expand Down Expand Up @@ -763,13 +772,15 @@ $(GOLANGCI_LINT_BIN): $(GOLANGCI_LINT) ## Build a local copy of golangci-lint.
.PHONY: $(GOVULNCHECK_BIN)
$(GOVULNCHECK_BIN): $(GOVULNCHECK) ## Build a local copy of govulncheck.

.PHONY: $(KUBE_STATE_METRICS_BIN)
$(KUBE_STATE_METRICS_BIN): $(KUBE_STATE_METRICS) ## Build a local copy of metric-gen.

.PHONY: $(GOVC_BIN)
$(GOVC_BIN): $(GOVC) ## Build a local copy of govc.

.PHONY: $(KIND_BIN)
$(KIND_BIN): $(KIND) ## Build a local copy of kind.


.PHONY: $(RELEASE_NOTES_BIN)
$(RELEASE_NOTES_BIN): $(RELEASE_NOTES) ## Build a local copy of release-notes.

Expand Down Expand Up @@ -815,15 +826,17 @@ $(GOLANGCI_LINT): # Build golangci-lint.
$(GOVULNCHECK): # Build govulncheck.
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOVULNCHECK_PKG) $(GOVULNCHECK_BIN) $(GOVULNCHECK_VER)

$(KUBE_STATE_METRICS): # Build kube-state-metrics.
GOBIN=$(TOOLS_BIN_DIR) GOMOD_REPLACE="$(KUBE_STATE_METRICS_MOD_REPLACE)" $(GO_TOOLS_BUILD) $(KUBE_STATE_METRICS_PKG) $(KUBE_STATE_METRICS_BIN) $(KUBE_STATE_METRICS_VER)

$(GOVC): # Build GOVC.
CGO_ENABLED=0 GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOVC_PKG) $(GOVC_BIN) $(GOVC_VER)

$(KIND): # Build kind.
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KIND_PKG) $(KIND_BIN) $(KIND_VER)


$(RELEASE_NOTES): # Build release-notes.
GOBIN=$(TOOLS_BIN_DIR) $(GO_TOOLS_BUILD) $(RELEASE_NOTES_PKG) $(RELEASE_NOTES_BIN) $(RELEASE_NOTES_VER)
GOBIN=$(TOOLS_BIN_DIR) GOMOD_REQUIRE="$(RELEASE_NOTES_MOD_REQUIRE)" $(GO_TOOLS_BUILD) $(RELEASE_NOTES_PKG) $(RELEASE_NOTES_BIN) $(RELEASE_NOTES_VER)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
GOBIN=$(TOOLS_BIN_DIR) GOMOD_REQUIRE="$(RELEASE_NOTES_MOD_REQUIRE)" $(GO_TOOLS_BUILD) $(RELEASE_NOTES_PKG) $(RELEASE_NOTES_BIN) $(RELEASE_NOTES_VER)
GOBIN=$(TOOLS_BIN_DIR) GO_MOD_REQUIRE="$(RELEASE_NOTES_MOD_REQUIRE)" $(GO_TOOLS_BUILD) $(RELEASE_NOTES_PKG) $(RELEASE_NOTES_BIN) $(RELEASE_NOTES_VER)

I think (?)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope. Here we need GO_MOD_REQUIRE, otherwise the commands would have a failure and the script won't finish.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My suggestion is to rename the variable :) Sorry wasn't pretty clear

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah :D 👍


## --------------------------------------
## Helpers
Expand Down
11 changes: 11 additions & 0 deletions apis/v1beta1/vspherecluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ type VSphereClusterStatus struct {

// Conditions defines current service state of the VSphereCluster.
// +optional
// +Metrics:stateset:name="status_condition",help="The condition of a vspherecluster.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspherecluster.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
Conditions clusterv1.Conditions `json:"conditions,omitempty"`

// FailureDomains is a list of failure domain objects synced from the infrastructure provider.
Expand All @@ -110,6 +112,15 @@ type VSphereClusterStatus struct {
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"

// VSphereCluster is the Schema for the vsphereclusters API
// +Metrics:gvk:namePrefix="capi_vspherecluster"
// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspherecluster is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
// +Metrics:info:name="info",help="Information about a vspherecluster.",labelsFromPath={status_vsphere_version:.status.vCenterVersion,spec_server:.spec.server,identity_reference_kind:".spec.identityRef.kind",identity_reference_name:".spec.identityRef.name"}
type VSphereCluster struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Expand Down
11 changes: 11 additions & 0 deletions apis/v1beta1/vsphereclusteridentity_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ type VSphereClusterIdentityStatus struct {

// Conditions defines current service state of the VSphereCluster.
// +optional
// +Metrics:stateset:name="status_condition",help="The condition of a vsphereclusteridentity.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vsphereclusteridentity.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
Conditions clusterv1.Conditions `json:"conditions,omitempty"`
}

Expand Down Expand Up @@ -85,6 +87,15 @@ func (c *VSphereClusterIdentity) SetConditions(conditions clusterv1.Conditions)
// +kubebuilder:subresource:status

// VSphereClusterIdentity defines the account to be used for reconciling clusters
// +Metrics:gvk:namePrefix="capi_vsphereclusteridentity"
// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vsphereclusteridentity is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
// +Metrics:info:name="info",help="Information about a vsphereclusteridentity.",labelsFromPath={name:.metadata.name}
type VSphereClusterIdentity struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Expand Down
11 changes: 11 additions & 0 deletions apis/v1beta1/vspheredeploymentzone_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ type VSphereDeploymentZoneStatus struct {

// Conditions defines current service state of the VSphereMachine.
// +optional
// +Metrics:stateset:name="status_condition",help="The condition of a vspheredeploymentzone.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspheredeploymentzone.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
Conditions clusterv1.Conditions `json:"conditions,omitempty"`
}

Expand All @@ -90,6 +92,15 @@ type VSphereDeploymentZoneStatus struct {
// +kubebuilder:subresource:status

// VSphereDeploymentZone is the Schema for the vspheredeploymentzones API
// +Metrics:gvk:namePrefix="capi_vspheredeploymentzone"
// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspheredeploymentzone is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
// +Metrics:info:name="info",help="Information about a vspheredeploymentzone.",labelsFromPath={name:.metadata.name}
type VSphereDeploymentZone struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Expand Down
9 changes: 9 additions & 0 deletions apis/v1beta1/vspherefailuredomain_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,15 @@ type FailureDomainHosts struct {
// +kubebuilder:resource:path=vspherefailuredomains,scope=Cluster,categories=cluster-api

// VSphereFailureDomain is the Schema for the vspherefailuredomains API
// +Metrics:gvk:namePrefix="capi_vspherefailuredomain"
// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspherefailuredomain is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
// +Metrics:info:name="info",help="Information about a vspherefailuredomain.",labelsFromPath={name:.metadata.name}
type VSphereFailureDomain struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Expand Down
12 changes: 12 additions & 0 deletions apis/v1beta1/vspheremachine_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ type VSphereMachineStatus struct {
Ready bool `json:"ready"`

// Addresses contains the VSphere instance associated addresses.
// +Metrics:info:name="status_addresses",help="Address information about a vspheremachine.",labelsFromPath={address:".address",type:".type"}
Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"`

// Network returns the network status for each of the machine's configured
Expand Down Expand Up @@ -125,6 +126,8 @@ type VSphereMachineStatus struct {

// Conditions defines current service state of the VSphereMachine.
// +optional
// +Metrics:stateset:name="status_condition",help="The condition of a vspheremachine.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspheremachine.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
Conditions clusterv1.Conditions `json:"conditions,omitempty"`
}

Expand All @@ -139,6 +142,15 @@ type VSphereMachineStatus struct {
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"

// VSphereMachine is the Schema for the vspheremachines API
// +Metrics:gvk:namePrefix="capi_vspheremachine"
// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspheremachine is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
// +Metrics:info:name="info",help="Information about a vspheremachine.",labelsFromPath={provider_id:.spec.providerID}
type VSphereMachine struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Expand Down
12 changes: 12 additions & 0 deletions apis/v1beta1/vspherevm_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ type VSphereVMStatus struct {
// This field is required at runtime for other controllers that read
// this CRD as unstructured data.
// +optional
// +Metrics:info:name="status_addresses",help="Address information about a vspherevm.",labelsFromPath={address:"."}
Addresses []string `json:"addresses,omitempty"`

// CloneMode is the type of clone operation used to clone this VM. Since
Expand Down Expand Up @@ -165,6 +166,8 @@ type VSphereVMStatus struct {

// Conditions defines current service state of the VSphereVM.
// +optional
// +Metrics:stateset:name="status_condition",help="The condition of a vspherevm.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a vspherevm.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
Conditions clusterv1.Conditions `json:"conditions,omitempty"`

// ModuleUUID is the unique identifier for the vCenter cluster module construct
Expand All @@ -187,6 +190,15 @@ type VSphereVMStatus struct {
// +kubebuilder:subresource:status

// VSphereVM is the Schema for the vspherevms API
// +Metrics:gvk:namePrefix="capi_vspherevm"
// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
// +Metrics:labelFromPath:name="namespace",JSONPath=".metadata.namespace"
// +Metrics:labelFromPath:name="uid",JSONPath=".metadata.uid"
// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
// +Metrics:info:name="annotation_paused",JSONPath=.metadata.annotations['cluster\.x-k8s\.io/paused'],help="Whether the vspherevm is paused and any of its resources will not be processed by the controllers.",labelsFromPath={paused_value:"."}
// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
// +Metrics:info:name="info",help="Information about a vspherevm.",labelsFromPath={status_clonemode:.status.cloneMode,bootstrap_reference_kind:".spec.bootstrapRef.kind",bootstrap_reference_name:".spec.bootstrapRef.name",status_vmref:".status.vmRef"}
type VSphereVM struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Expand Down
14 changes: 11 additions & 3 deletions hack/go-tools-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ if [ -z "${GOBIN}" ]; then
exit 1
fi

CAPI_HACK_TOOLS="sigs.k8s.io/cluster-api/hack/tools"
export GOWORK="off"

GOMOD_REPLACE=${GOMOD_REPLACE:=}
Copy link
Member

@sbueringer sbueringer Sep 1, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: A bit of godoc around these variables would be good

GOMOD_REQUIRE=${GOMOD_REQUIRE:=}

rm -f "${GOBIN}/${2}"* || true

Expand All @@ -52,8 +55,13 @@ cd "${TMP_MODULE_DIR}"

# Initialize a go module and place a tools.go file for building the binary.
go mod init "tools"
# Set require for "sigs.k8s.io/cluster-api" to let go resolve the tools version via the CAPI version.
go mod edit -require "${CAPI_HACK_TOOLS}@${3}"

for PARAM in ${GOMOD_REPLACE}; do
eval go mod edit -replace "${PARAM}"
done
for PARAM in ${GOMOD_REQUIRE}; do
eval go mod edit -require "${PARAM}"
done

# Create go file which imports the required package and resolve dependencies.
cat << EOF > tools.go
Expand Down