Remove the SQLite DB, wkp-agent, event-writer and NATS (#790)

* Remove the SQLite database, agent and event-writer We've recently removed support for SQLite database. All state is now held in etcd/k8s. This also deprecates the whole agent model so we can remove that too. NATS and the event-writer were supporting that too. * Disable test checking 404 on cluster deletion - Git provider should show an error failing to delete no-existant files from git.
weaveworks · May 23, 2022 · 25da7e2 · 25da7e2
1 parent 496d0cd
commit 25da7e2
Show file tree

Hide file tree

Showing 126 changed files with 132 additions and 13,497 deletions.
diff --git a/.github/workflows/acceptance-test.yaml b/.github/workflows/acceptance-test.yaml
@@ -23,7 +23,7 @@ on:
       label-filter:
         description: "Which tests to filter -- eg: --label-filter='smoke' or --label-filter='@smoke && !integration'"
         required: true
-        type: string 
+        type: string
       kubectl-version:
         description: "Version of kubectl"
         required: true
@@ -58,11 +58,6 @@ on:
         default: "/usr/local/bin/gitops"
         required: false
         type: string
-      database-type:
-        description: "Acceptance tests database DB type e.g. sqlite or postgres"
-        default: sqlite
-        required: false
-        type: string
       artifacts-base-dir:
         description: "Directory for test artifacts"
         required: true
@@ -75,7 +70,7 @@ on:
         description: "GCE leaf cluster kubeconfig path (for connect cluster tests)"
         required: false
         type: string
-    secrets:      
+    secrets:
       WGE_CLUSTER_ADMIN_PASSWORD:
         description: "Cluster user admin password"
         required: true
@@ -84,7 +79,7 @@ on:
         required: true
       WGE_DEX_CLIENT_SECRET:
         description: "client credential secret for OIDC (dex)"
-        required: true      
+        required: true
       WGE_GIT_PROVIDER_PRIVATE_KEY:
         description: "Key for accessing git provider e.g. github or gitlab"
         required: false
@@ -162,13 +157,11 @@ env:
   GITLAB_CLIENT_ID: ${{ secrets.WGE_GITLAB_CLIENT_ID }}
   GITLAB_CLIENT_SECRET: ${{ secrets.WGE_GITLAB_CLIENT_SECRET }}
   UI_NODEPORT: 30080
-  NATS_NODEPORT: 31490
   MANAGEMENT_CLUSTER_CNAME: weave.gitops.enterprise.com
   UPGRADE_MANAGEMENT_CLUSTER_CNAME: weave.gitops.upgrade.enterprise.com
   MANAGEMENT_CLUSTER_KIND: ${{ inputs.management-cluster-kind }}
   CAPI_PROVIDER: ${{ inputs.capi_provider }}
   GITOPS_BIN_PATH: ${{ inputs.gitops-bin-path }}
-  ACCEPTANCE_TESTS_DATABASE_TYPE: ${{ inputs.database-type }}
   EKS_LEAF_KUBECONFIG: ${{ inputs.eks-leaf-kubeconfig }}
   WGE_ACCEPTANCE_EKS_KUBECONFIG: ${{ secrets.WGE_ACCEPTANCE_EKS_KUBECONFIG }}
   GCE_LEAF_KUBECONFIG: ${{ inputs.gce-leaf-kubeconfig }}

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -100,31 +100,6 @@ jobs:
       - name: Push image
         run: docker push docker.io/weaveworks/weave-gitops-enterprise-clusters-service:$(./tools/image-tag)
 
-  event-writer:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Configure git for private modules
-        run: git config --global url."https://${GITHUB_BUILD_USERNAME}:${GITHUB_BUILD_TOKEN}@github.com".insteadOf "https://github.com"
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Grab all tags from git
-        run: |
-          git fetch --tags -f
-      - name: build
-        run: |
-          make GITHUB_BUILD_TOKEN=${GITHUB_BUILD_TOKEN} cmd/event-writer/.uptodate
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          registry: docker.io
-          username: ${{ secrets.WGE_DOCKER_IO_USER }}
-          password: ${{ secrets.WGE_DOCKER_IO_PASSWORD }}
-      - name: Push images to docker registry
-        run: |
-          docker push docker.io/weaveworks/weave-gitops-enterprise-event-writer:$(./tools/image-tag)
-
   ui-server:
     runs-on: ubuntu-latest
     steps:
@@ -147,25 +122,3 @@ jobs:
       - name: Push images to docker registry
         run: |
           docker push docker.io/weaveworks/weave-gitops-enterprise-ui-server:$(./tools/image-tag)
-
-  gitops-agent:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Grab all tags from git
-        run: |
-          git fetch --tags -f
-      - name: Build the agent
-        run: |
-          make GITHUB_BUILD_TOKEN=${GITHUB_BUILD_TOKEN} cmd/wkp-agent/.uptodate
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          registry: docker.io
-          username: ${{ secrets.WGE_DOCKER_IO_USER }}
-          password: ${{ secrets.WGE_DOCKER_IO_PASSWORD }}
-      - name: Push images to docker registry
-        run: docker push docker.io/weaveworks/wkp-agent:$(./tools/image-tag)
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -26,46 +26,45 @@ jobs:
       WGE_COVERALLS_TOKEN: ${{ secrets.WGE_COVERALLS_TOKEN }}
       ARTEFACTS_BASE_DIR: /tmp/workspace/test
     steps:
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.18.x
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Configure git for private modules
-      env:
-        GITHUB_BUILD_USERNAME: wge-build-bot
-        GITHUB_BUILD_TOKEN: ${{ secrets.BUILD_BOT_PERSONAL_ACCESS_TOKEN }}
-      run: git config --global url."https://${GITHUB_BUILD_USERNAME}:${GITHUB_BUILD_TOKEN}@github.com".insteadOf "https://github.com"
-    - name: Install dependencies
-      run: |
-        go mod download
-        go install github.com/wadey/gocovmerge@latest
-        go install github.com/jstemmer/go-junit-report@latest
-        npm install -g junit-report-merger
-    - name: Install goveralls
-      env:
-        GO111MODULE: off
-      run: go get github.com/mattn/goveralls
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.x
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Configure git for private modules
+        env:
+          GITHUB_BUILD_USERNAME: wge-build-bot
+          GITHUB_BUILD_TOKEN: ${{ secrets.BUILD_BOT_PERSONAL_ACCESS_TOKEN }}
+        run: git config --global url."https://${GITHUB_BUILD_USERNAME}:${GITHUB_BUILD_TOKEN}@github.com".insteadOf "https://github.com"
+      - name: Install dependencies
+        run: |
+          go mod download
+          go install github.com/wadey/gocovmerge@latest
+          go install github.com/jstemmer/go-junit-report@latest
+          npm install -g junit-report-merger
+      - name: Install goveralls
+        env:
+          GO111MODULE: off
+        run: go get github.com/mattn/goveralls
 
-    - name: Run unit tests
-      run: |
-        go version
-        mkdir -p ${{ env.ARTEFACTS_BASE_DIR }}
+      - name: Run unit tests
+        run: |
+          go version
+          mkdir -p ${{ env.ARTEFACTS_BASE_DIR }}
 
-        WKP_DEBUG=true go test -cover -coverprofile=.coverprofile ./cmd/... ./pkg/... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/test-results.xml
-        cd ${{ github.workspace }}/cmd/event-writer && go test -cover -coverprofile=.coverprofile ./... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/event-writer-results.xml
-        cd ${{ github.workspace }}/common && go test -cover -coverprofile=.coverprofile ./... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/common-results.xml
-        cd ${{ github.workspace }}/cmd/clusters-service && go test -cover -coverprofile=.coverprofile ./... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/clusters-service-results.xml
+          WKP_DEBUG=true go test -cover -coverprofile=.coverprofile ./cmd/... ./pkg/... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/test-results.xml
+          cd ${{ github.workspace }}/common && go test -cover -coverprofile=.coverprofile ./... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/common-results.xml
+          cd ${{ github.workspace }}/cmd/clusters-service && go test -cover -coverprofile=.coverprofile ./... | go-junit-report > ${{ env.ARTEFACTS_BASE_DIR }}/clusters-service-results.xml
 
-        cd ${{ github.workspace }}
-        # Merge all coverage results
-        gocovmerge .coverprofile cmd/event-writer/.coverprofile common/.coverprofile  cmd/clusters-service/.coverprofile > ${{ env.ARTEFACTS_BASE_DIR }}/merged-profiles    
-        # Merge all junit test results
-        jrm ${{ env.ARTEFACTS_BASE_DIR }}/combined-test-results.xml '${{ env.ARTEFACTS_BASE_DIR }}/*.xml'
-    - name: Store unit test coverage results
-      uses: actions/upload-artifact@v2
-      with:
+          cd ${{ github.workspace }}
+          # Merge all coverage results
+          gocovmerge .coverprofile common/.coverprofile  cmd/clusters-service/.coverprofile > ${{ env.ARTEFACTS_BASE_DIR }}/merged-profiles    
+          # Merge all junit test results
+          jrm ${{ env.ARTEFACTS_BASE_DIR }}/combined-test-results.xml '${{ env.ARTEFACTS_BASE_DIR }}/*.xml'
+      - name: Store unit test coverage results
+        uses: actions/upload-artifact@v2
+        with:
           name: unit-tests-artifacts
           path: |
             ${{ env.ARTEFACTS_BASE_DIR }}
@@ -76,7 +75,7 @@ jobs:
     with:
       runs-on: ubuntu-latest
       ref: main
-  
+
   acceptance-tests-gitlab-on-prem:
     needs: [build, coverage, gitops-binary]
     uses: ./.github/workflows/acceptance-test.yaml
@@ -93,7 +92,6 @@ jobs:
       management-cluster-kind: kind
       capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: sqlite
       artifacts-base-dir: "/tmp/acceptance-test-gitlab-on-prem"
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"

diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -39,7 +39,6 @@ jobs:
       management-cluster-kind: eks
       capi_provider: capa
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: postgres
       eks-leaf-kubeconfig: "/tmp/eks-leaf-kubeconfig"
       artifacts-base-dir: "/tmp/acceptance-test-eks-github"
     secrets:
@@ -78,10 +77,9 @@ jobs:
       management-cluster-kind: gke
       capi_provider: capg
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: postgres
       gce-leaf-kubeconfig: "/tmp/gce-leaf-kubeconfig"
       artifacts-base-dir: "/tmp/acceptance-test-gke-gitlab"
-    secrets:      
+    secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
       WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
       WGE_DEX_CLIENT_SECRET: ${{ secrets.WGE_DEX_CLIENT_SECRET }}

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -150,7 +150,6 @@ jobs:
       - name: Run integration tests
         run: |
           go version
-          go test -v ./test/integration/test -run TestAgent
 
           cd cmd/clusters-service
           go test -v ./... -tags=integration
@@ -239,7 +238,6 @@ jobs:
       management-cluster-kind: kind
       capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: sqlite
       artifacts-base-dir: "/tmp/smoke-test-github"
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
@@ -268,7 +266,6 @@ jobs:
       management-cluster-kind: kind
       capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: sqlite
       artifacts-base-dir: "/tmp/smoke-test-gitlab"
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
@@ -299,7 +296,6 @@ jobs:
       management-cluster-kind: kind
       capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: sqlite
       artifacts-base-dir: "/tmp/smoke-test-gitlab-on-prem"
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
@@ -336,7 +332,6 @@ jobs:
       management-cluster-kind: kind
       capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
-      database-type: sqlite
       artifacts-base-dir: "/tmp/acceptance-test-github"
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
@@ -348,4 +343,3 @@ jobs:
       WGE_GITHUB_USER: ${{ secrets.WGE_GITHUB_USER }}
       WGE_GITHUB_PASSWORD: ${{ secrets.WGE_GITHUB_PASSWORD }}
       WGE_GITHUB_MFA_KEY: ${{ secrets.WGE_GITHUB_MFA_KEY }}
-
diff --git a/.gitignore b/.gitignore
@@ -12,19 +12,6 @@
 # Recovery files from vim
 *.swp
 
-# Ignore generated binaries
-/cmd/k8s-krb5-server/server
-/cmd/mock-authz-server/server
-/cmd/mock-https-authz-server/server
-/cmd/policy/policy
-/cmd/update-manifest/update-manifest
-/cmd/wk/wk
-/cmd/wks-ci/checks/policy/policy
-/cmd/wks-ci/wks-ci
-/cmd/wks-controller/controller
-/cmd/wks-entitle/wks-entitle
-/cmd/wkp-agent/wkp-agent
-
 node_modules
 
 # Ignore binary dependencies

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -12,17 +12,6 @@ Weave GitOps Enterprise (WGE) is packaged as a Helm chart and currently consists
   Allows for custom Jobs to be executed on newly provisioned CAPI clusters. Most often, this will be used to install CNI which CAPI does not install. Without this controller, newly provisioned clusters would not be ready to be used by end users. Because it also references the CAPI CRD, it requires CAPI tooling to be installed first.
 - [cluster-controller](https://github.com/weaveworks/cluster-controller)
   Defines the CRD for declaring leaf clusters. A leaf cluster is a cluster that the management cluster can query via a kubeconfig. This controller ensures that kubeconfig secrets have been supplied for leaf clusters. Because it also references the CAPI CRD, it requires CAPI tooling to be installed first.
-- event-writer **Soon to be deprecated**
-
-  Subscribes to a NATS queue and listens for messages sent by the wkp-agent. These messages carry cluster information about leaf clusters which is then written to a file-based SQLite db.
-- wkp-agent **Soon to be deprecated**
-
-  Agent installed by end users to management cluster and leaf clusters with the purpose of reporting back cluster information to management cluster.
-
-As part of the chart, other dependencies also get installed:
-- NATS **Soon to be deprecated**
-
-  Agents running on the management cluster or on leaf clusters send events to a NATS queue. event-writer receives these events from the queue and does its thing.
 
 ## One-time setup
 You need a github Personal Access Token to build the service. This token needs at least the `repo` and `read:packages` permissions. If you want to be able to delete the GitOps repo every time you recreate your local Kind cluster, add the `delete_repo` permission too and set the `DELETE_GITOPS_DEV_REPO` flag to 1.  You can create a token [here](https://github.com/settings/tokens), and export it as:
@@ -81,7 +70,6 @@ Any change you make to local code will trigger tilt to rebuild and restart the p
 
 - If a change in your local settings results in a ConfigMap update, you will need to restart the `clusters-service` pod in order for the pod to read the updated ConfigMap.
 - Every time you restart `clusters-service` it will generate new self-signed certificates, therefore you will need to reload the UI and accept the new certificate. Check for TLS certificate errors in the `chart-mccp-cluster-service` logs and if necessary re-trigger an update to rebuild it.
-- The `wkp-agent` image tag is currently hard-coded to `v0.8.0-rc.1` and by default Tilt won't build/deploy it. The benefit of that is the ability to easily install it on your local Kind cluster and view CAPI status for any clusters you create.
 
 ### Faster frontend development
 Especially for frontend development, the time it takes for the pod to restart can be annoying. To spin up a local development frontend against your development cluster, run:
@@ -244,34 +232,13 @@ Sometimes it's nice to demo / experiment with the service(s) you're changing loc
 
 ### The `clusters-service`
 
-_Note: the following instructions will use a new local database, you can probably reconcile the internal cluster database with the local one with some fancy fs mounting, tbd..._
-
 To have entitlements, create a cluster and point your `kubectl` to it. It doesn't matter what kind of cluster you create.
 Integration tests have a config located [here](../test/integration/test/kind-config.yaml) for inspiration.
 
 The `clusters-service` requires the presence of a valid entitlement secret for it to work. Make sure an entitlement secret has been added to the cluster and that the `clusters-service` has been configured to look for it using the correct namespace/name. By default, entitlement secrets are named `weave-gitops-enterprise-credentials` and are added to the `flux-system` namespace. If that's not the case, you will need to point the service to the right place by explicitly specifying the relevant environment variables (example below).
 
 An existing entitlement secret that you can use can be found [here](../test/utils/scripts/entitlement-secret.yaml). Alternatively, you can generate your own entitlement secret by using the `wge-credentials` binary.
 
-#### Create a local database (optional):
-
-```bash
-$ (cd cmd/event-writer && go run main.go database create --db-type sqlite --db-uri file:///tmp/wge.db)
-INFO[0000] created all database tables
-
-# inspect db
-$ sqlite3 /tmp/wge.db
-SQLite version 3.28.0 2019-04-15 14:49:49
-Enter ".help" for usage hints.
-
-sqlite> .tables
-alerts                 cluster_statuses       git_commits
-capi_clusters          clusters               node_info
-cluster_info           events                 pull_requests
-cluster_pull_requests  flux_info              workspaces
-sqlite>
-```
-
 #### Port forward the source-controller to access profiles (optional):
 
 To query profiles the `clusters-service` needs to be able to DNS resolve the source-controller which provides the helm-repo (profile) info.
@@ -306,7 +273,7 @@ export WEAVE_GITOPS_FLUX_BIN_PATH=`which flux`
 SOURCE_CONTROLLER_LOCALHOST=localhost:8080
 
 # Run the server configured using lots of env vars
-DB_URI=/tmp/wge.db CAPI_CLUSTERS_NAMESPACE=default CAPI_TEMPLATES_NAMESPACE=default GIT_PROVIDER_TYPE=github GIT_PROVIDER_HOSTNAME=github.com CAPI_TEMPLATES_REPOSITORY_URL=https://github.com/my-org/my-repo CAPI_TEMPLATES_REPOSITORY_BASE_BRANCH=main ENTITLEMENT_SECRET_NAMESPACE=flux-system ENTITLEMENT_SECRET_NAME=weave-gitops-enterprise-credentials go run cmd/clusters-service/main.go
+CAPI_CLUSTERS_NAMESPACE=default CAPI_TEMPLATES_NAMESPACE=default GIT_PROVIDER_TYPE=github GIT_PROVIDER_HOSTNAME=github.com CAPI_TEMPLATES_REPOSITORY_URL=https://github.com/my-org/my-repo CAPI_TEMPLATES_REPOSITORY_BASE_BRANCH=main ENTITLEMENT_SECRET_NAMESPACE=flux-system ENTITLEMENT_SECRET_NAME=weave-gitops-enterprise-credentials go run cmd/clusters-service/main.go
 ```
 
 You can query the local capi-server:
@@ -519,24 +486,6 @@ helm repo update && helm search repo wkp --devel --versions | grep e4e540d
 ```
 where `e4e540d` is your commit sha. This will return `wkp/mccp      0.0.17-88-ge4e540d      1.16.0          A Helm chart for Kubernetes` where `0.0.17-88-ge4e540d` is the version you're looking for.
 
-
-## How to inspect/modify the `sqlite` database of a running cluster
-
-Copy the database to your local machine and inspect using sqlite
-
-```bash
-kubectl cp mccp/mccp-cluster-service-79854d9fcb-bwvp7:/var/database/mccp.db mccp.db
-sqlite mccp.db
-```
-
-Or, we can inspect _and modify_ the database in the cluster with
-
-```bash
-kubectl exec -ti -n mccp mccp-cluster-service-79854d9fcb-bwvp7 -- /bin/sh
-apk add sqlite3
-sqlite /var/database/mccp.db
-```
-
 ## How to make a self-signed cert that works in chrome!
 
 ```bash