Skip to content

Commit

Permalink
makefile consistency (#426)
Browse files Browse the repository at this point in the history
  • Loading branch information
jpinsonneau authored May 11, 2023
1 parent 95395fc commit 5f57707
Show file tree
Hide file tree
Showing 12 changed files with 332 additions and 243 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/push_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ on:

env:
REGISTRY_USER: netobserv+github_ci
REGISTRY_PASSWORD: ${{ secrets.QUAY_SECRET }}
REGISTRY: quay.io/netobserv
IMAGE: flowlogs-pipeline
TAG: main
ORG: netobserv
VERSION: main

jobs:
push-image:
Expand All @@ -30,7 +30,7 @@ jobs:
uses: docker/login-action@v2
with:
username: ${{ env.REGISTRY_USER }}
password: ${{ env.REGISTRY_PASSWORD }}
password: ${{ secrets.QUAY_SECRET }}
registry: quay.io
- name: build and push images
run: OCI_RUNTIME=docker DOCKER_TAG=${{ env.TAG }} make push-ci-images
- name: build and push manifest with images
run: IMAGE_ORG=${{ env.ORG }} OCI_BIN=docker VERSION=${{ env.VERSION }} make ci
7 changes: 5 additions & 2 deletions .github/workflows/push_image_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ env:
REGISTRY_USER: netobserv+github_ci
REGISTRY: quay.io/netobserv
IMAGE: flowlogs-pipeline
ORG: netobserv
VERSION: temp

jobs:
push-pr-image:
Expand Down Expand Up @@ -35,9 +37,10 @@ jobs:
registry: quay.io
- name: get short sha
run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV

- name: build and push images
run: OCI_RUNTIME=docker DOCKER_TAG=temp make push-ci-images
run: IMAGE_ORG=${{ env.ORG }} OCI_BIN=docker IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ env.short_sha }} make images
- name: build and push manifest
run: IMAGE_ORG=${{ env.ORG }} OCI_BIN=docker VERSION=${{ env.VERSION }} make ci-manifest
- uses: actions/github-script@v6
with:
github-token: ${{secrets.GITHUB_TOKEN}}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,4 @@ jobs:
password: ${{ env.REGISTRY_PASSWORD }}
registry: quay.io
- name: build and push images
run: OCI_RUNTIME=docker DOCKER_TAG=${{ env.tag }} make push-multiarch-manifest
run: OCI_BIN=docker VERSION=${{ env.tag }} make images
156 changes: 156 additions & 0 deletions .mk/development.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
##@ kubernetes

# note: to deploy with custom image tag use: VERSION=test make deploy
.PHONY: deploy
deploy: ## Deploy the image
sed 's|%IMAGE_TAG_BASE%|$(IMAGE_TAG_BASE)|g;s|%VERSION%|$(VERSION)|g' contrib/kubernetes/deployment.yaml > /tmp/deployment.yaml
kubectl create configmap flowlogs-pipeline-configuration --from-file=flowlogs-pipeline.conf.yaml=$(FLP_CONF_FILE)
kubectl apply -f /tmp/deployment.yaml
kubectl rollout status "deploy/flowlogs-pipeline" --timeout=600s

.PHONY: undeploy
undeploy: ## Undeploy the image
sed 's|%IMAGE_TAG_BASE%|$(IMAGE_TAG_BASE)|g;s|%VERSION%|$(VERSION)|g' contrib/kubernetes/deployment.yaml > /tmp/deployment.yaml
kubectl --ignore-not-found=true delete configmap flowlogs-pipeline-configuration || true
kubectl --ignore-not-found=true delete -f /tmp/deployment.yaml || true

.PHONY: deploy-loki
deploy-loki: ## Deploy loki
kubectl apply -f contrib/kubernetes/deployment-loki-storage.yaml
kubectl apply -f contrib/kubernetes/deployment-loki.yaml
kubectl rollout status "deploy/loki" --timeout=600s
-pkill --oldest --full "3100:3100"
kubectl port-forward --address 0.0.0.0 svc/loki 3100:3100 2>&1 >/dev/null &
@echo -e "\nloki endpoint is available on http://localhost:3100\n"

.PHONY: undeploy-loki
undeploy-loki: ## Undeploy loki
kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-loki.yaml || true
kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-loki-storage.yaml || true
-pkill --oldest --full "3100:3100"

.PHONY: deploy-prometheus
deploy-prometheus: ## Deploy prometheus
kubectl apply -f contrib/kubernetes/deployment-prometheus.yaml
kubectl rollout status "deploy/prometheus" --timeout=600s
-pkill --oldest --full "9090:9090"
kubectl port-forward --address 0.0.0.0 svc/prometheus 9090:9090 2>&1 >/dev/null &
@echo -e "\nprometheus ui is available on http://localhost:9090\n"

.PHONY: undeploy-prometheus
undeploy-prometheus: ## Undeploy prometheus
kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-prometheus.yaml || true
-pkill --oldest --full "9090:9090"

.PHONY: deploy-grafana
deploy-grafana: ## Deploy grafana
kubectl create configmap grafana-dashboard-definitions --from-file=contrib/dashboards/
kubectl apply -f contrib/kubernetes/deployment-grafana.yaml
kubectl rollout status "deploy/grafana" --timeout=600s
-pkill --oldest --full "3000:3000"
kubectl port-forward --address 0.0.0.0 svc/grafana 3000:3000 2>&1 >/dev/null &
@echo -e "\ngrafana ui is available (user: admin password: admin) on http://localhost:3000\n"

.PHONY: undeploy-grafana
undeploy-grafana: ## Undeploy grafana
kubectl --ignore-not-found=true delete configmap grafana-dashboard-definitions || true
kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-grafana.yaml || true
-pkill --oldest --full "3000:3000"

.PHONY: deploy-netflow-simulator
deploy-netflow-simulator: ## Deploy netflow simulator
kubectl apply -f contrib/kubernetes/deployment-netflow-simulator.yaml
kubectl rollout status "deploy/netflowsimulator" --timeout=600s

.PHONY: undeploy-netflow-simulator
undeploy-netflow-simulator: ## Undeploy netflow simulator
kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-netflow-simulator.yaml || true

##@ kind

.PHONY: create-kind-cluster
create-kind-cluster: $(KIND) ## Create cluster
$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --config contrib/kubernetes/kind/kind.config.yaml
kubectl cluster-info --context kind-kind

.PHONY: delete-kind-cluster
delete-kind-cluster: $(KIND) ## Delete cluster
$(KIND) delete cluster --name $(KIND_CLUSTER_NAME)

.PHONY: kind-load-image
kind-load-image: ## Load image to kind
ifeq ($(OCI_BIN),$(shell which docker))
# This is an optimization for docker provider. "kind load docker-image" can load an image directly from docker's
# local registry. For other providers (i.e. podman), we must use "kind load image-archive" instead.
$(KIND) load --name $(KIND_CLUSTER_NAME) docker-image $(IMAGE)-${GOARCH}
else
$(eval tmpfile="/tmp/flp.tar")
-rm $(tmpfile)
$(OCI_BIN) save $(IMAGE)-$(GOARCH) -o $(tmpfile)
$(KIND) load --name $(KIND_CLUSTER_NAME) image-archive $(tmpfile)
-rm $(tmpfile)
endif

##@ metrics

.PHONY: generate-configuration
generate-configuration: $(KIND) ## Generate metrics configuration
go build "${CMD_DIR}${CG_BIN_FILE}"
./${CG_BIN_FILE} --log-level debug --srcFolder network_definitions \
--destConfFile $(FLP_CONF_FILE) \
--destDocFile docs/metrics.md \
--destGrafanaJsonnetFolder contrib/dashboards/jsonnet/

##@ End2End

.PHONY: local-deployments-deploy
local-deployments-deploy: $(KIND) deploy-prometheus deploy-loki deploy-grafana build-image kind-load-image deploy deploy-netflow-simulator
kubectl get pods
kubectl rollout status -w deployment/flowlogs-pipeline
kubectl logs -l app=flowlogs-pipeline

.PHONY: local-deploy
local-deploy: $(KIND) local-cleanup create-kind-cluster local-deployments-deploy ## Deploy locally on kind (with simulated flowlogs)

.PHONY: local-deployments-cleanup
local-deployments-cleanup: $(KIND) undeploy-netflow-simulator undeploy undeploy-grafana undeploy-loki undeploy-prometheus

.PHONY: local-cleanup
local-cleanup: $(KIND) local-deployments-cleanup delete-kind-cluster ## Undeploy from local kind

.PHONY: local-redeploy
local-redeploy: local-deployments-cleanup local-deployments-deploy ## Redeploy locally (on current kind)

.PHONY: ocp-deploy
ocp-deploy: ocp-cleanup deploy-prometheus deploy-loki deploy-grafana deploy ## Deploy to OCP
flowlogs_pipeline_svc_ip=$$(kubectl get svc flowlogs-pipeline -o jsonpath='{.spec.clusterIP}'); \
./hack/enable-ocp-flow-export.sh $$flowlogs_pipeline_svc_ip
kubectl get pods
kubectl rollout status -w deployment/flowlogs-pipeline
kubectl logs -l app=flowlogs-pipeline
oc expose service prometheus || true
@prometheus_url=$$(oc get route prometheus -o jsonpath='{.spec.host}'); \
echo -e "\nAccess prometheus on OCP using: http://"$$prometheus_url"\n"
oc expose service grafana || true
@grafana_url=$$(oc get route grafana -o jsonpath='{.spec.host}'); \
echo -e "\nAccess grafana on OCP using: http://"$$grafana_url"\n"
oc expose service loki || true
@loki_url=$$(oc get route loki -o jsonpath='{.spec.host}'); \
echo -e "\nAccess loki on OCP using: http://"$$loki_url"\n"

.PHONY: ocp-cleanup
ocp-cleanup: undeploy undeploy-loki undeploy-prometheus undeploy-grafana ## Undeploy from OCP

.PHONY: dev-local-deploy
dev-local-deploy: ## Deploy locally with simulated netflows
-pkill --oldest --full "${FLP_BIN_FILE}"
-pkill --oldest --full "${NETFLOW_GENERATOR}"
go build "${CMD_DIR}${FLP_BIN_FILE}"
go build "${CMD_DIR}${CG_BIN_FILE}"
./${CG_BIN_FILE} --log-level debug --srcFolder network_definitions \
--skipWithTags "kubernetes" \
--destConfFile /tmp/flowlogs-pipeline.conf.yaml --destGrafanaJsonnetFolder /tmp/
test -f /tmp/${NETFLOW_GENERATOR} || curl -L --output /tmp/${NETFLOW_GENERATOR} https://github.com/nerdalert/nflow-generator/blob/master/binaries/nflow-generator-x86_64-linux?raw=true
chmod +x /tmp/${NETFLOW_GENERATOR}
./"${FLP_BIN_FILE}" --config /tmp/flowlogs-pipeline.conf.yaml &
/tmp/${NETFLOW_GENERATOR} -t 127.0.0.1 -p 2056 > /dev/null 2>&1 &
28 changes: 28 additions & 0 deletions .mk/shortcuts.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
##@ shortcuts helpers

.PHONY: build-image
build-image: image-build ## Build MULTIARCH_TARGETS images

.PHONY: push-image
push-image: image-push ## Push MULTIARCH_TARGETS images

.PHONY: build-manifest
build-manifest: manifest-build ## Build MULTIARCH_TARGETS manifest

.PHONY: push-manifest
push-manifest: manifest-push ## Push MULTIARCH_TARGETS manifest

.PHONY: images
images: image-build image-push manifest-build manifest-push ## Build and push MULTIARCH_TARGETS images and related manifest

.PHONY: build-ci-manifest
build-ci-manifest: ci-manifest-build ## Build CI manifest

.PHONY: push-ci-manifest
push-ci-manifest: ci-manifest-push ## Push CI manifest

.PHONY: ci-manifest
ci-manifest: ci-manifest-build ci-manifest-push ## Build and push CI manifest

.PHONY: ci
ci: images ci-manifest ## Build and push CI images and manifest
Loading

0 comments on commit 5f57707

Please sign in to comment.