Doc updates

.github/workflows/checkov.yaml

@@ -3,6 +3,8 @@ name: Checkov
 on:
   push:
     branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
   workflow_dispatch:
 
 permissions: read-all
@@ -24,14 +26,14 @@ jobs:
         id: checkov
         uses: bridgecrewio/checkov-action@0549dc60bddd4c55cb85c6c3a07072e3cf2ca48e
         with:
-          skip_check: CKV_DOCKER_2,CKV_DOCKER_3
+          skip_check: CKV_DOCKER_2,CKV_DOCKER_3,CKV_SECRET_6
           quiet: true
           output_format: cli,sarif
           output_file_path: console,results.sarif
           download_external_modules: true
 
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3
         if: success() || failure()
         with:
           sarif_file: results.sarif

.github/workflows/helm_lint.yaml

@@ -0,0 +1,47 @@
+name: Lint and Test Chart
+
+on: pull_request
+
+permissions: read-all
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@20d2b4f98d41febe2bbca46408499dbb535b6258 # v3
+        with:
+          version: v3.14.0
+
+      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
+        with:
+          python-version: '3.12'
+          check-latest: true
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }}
+
+      - name: Create kind cluster
+        if: steps.list-changed.outputs.changed == 'true'
+        uses: helm/[email protected]
+
+      - name: Run chart-testing (install)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct install --target-branch ${{ github.event.repository.default_branch }}

.github/workflows/lint.yaml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
         with:
           python-version: '3.11'
       - name: "Upgrade pip"

.github/workflows/no_forgoten_migrations.yaml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
         with:
           python-version: '3.11'
       - name: "Upgrade pip"

.github/workflows/publish-and-deploy.yaml

@@ -3,14 +3,15 @@ name: Publish and Deploy
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, james/in_fra_ception ]
 
 permissions: read-all
 
 jobs:
   push_to_registry:
     name: Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -40,6 +41,7 @@ jobs:
     name: Deploy to grandsvc
     needs: push_to_registry
     runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
     steps:
     - name: Install SSH key
       uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # v2
@@ -61,3 +63,30 @@ jobs:
 
     - name: Pull new Docker image
       run: ssh ${{ secrets.GRANDSVC_SSH_USER }}@${{ secrets.GRANDSVC_TARGET_IP }} "cd ${{ secrets.GRANDSVC_PROJECT_PATH }} && git pull && docker compose pull && docker compose up -d"
+
+  deploy_to_dev0:
+    name: Deploy to dev3
+    #needs: push_to_registry
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/james/in_fra_ception'
+    steps:
+    - name: Install SSH key
+      uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # v2
+      with:
+        key: ${{ secrets.DEV3_KEY }}
+        name: id_ed25519 # optional
+        known_hosts: ${{ secrets.DEV3_KNOWN_HOSTS }}
+        #config: ${{ secrets.CONFIG }} # ssh_config; optional
+        if_key_exists: fail # replace / ignore / fail; optional (defaults to fail)
+
+    - name: Setup WireGuard
+      run:  |
+        sudo apt install wireguard
+        echo "${{ secrets.WIREGUARD_PRIVATE_KEY }}" > privatekey
+        sudo ip link add dev wg1 type wireguard
+        sudo ip address add dev wg1 ${{ secrets.WIREGUARD_OVERLAY_NETWORK_IP }} peer ${{ secrets.DEV3_TARGET_IP }}
+        sudo wg set wg1 listen-port 48123 private-key privatekey peer ${{ secrets.WIREGUARD_PEER_PUBLIC_KEY }} allowed-ips 0.0.0.0/0 endpoint ${{ secrets.WIREGUARD_ENDPOINT }}
+        sudo ip link set up dev wg1
+
+    - name: Pull new Docker image
+      run: ssh ${{ secrets.DEV3_SSH_USER }}@${{ secrets.DEV3_TARGET_IP }} "sudo bash -c 'cd ${{ secrets.DEV3_PROJECT_PATH }} && git pull && cd infra/helm/meshdb && helm template . -f ../../../../values.yaml -f  ../../../../secret.values.yaml | kubectl apply -f -'"

.github/workflows/run_django_tests.yaml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
         with:
           python-version: '3.11'
       - name: "Upgrade pip"

infra/.gitignore

@@ -0,0 +1,36 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+.terraform.lock.hcl

infra/README.md

@@ -0,0 +1,62 @@
+# Meshdb Environment Setup
+
+These instructions will set up a 4 node k3s cluster on proxmox.
+- 1 "manager" node for control plane and to be used for deployments.
+- 3 "agent" nodes to run services.
+
+1. Clone this repository
+
+2. Set up tfvars. See [proxmox provider](https://registry.terraform.io/providers/Telmate/proxmox/latest/docs). Create an API key in Proxmox, and disable Privilege Separation.
+```
+cd meshdb/infra/tf/
+cp example.tfvars your_env.tfvars
+# Modify your_env.tfvars to meet your needs
+bash gen_ssh_key.sh dev0
+```
+
+3. Create the k3s cluster
+```
+terraform init -var-file=your_env.tfvars
+terraform plan -var-file=your_env.tfvars
+terraform apply -var-file=your_env.tfvars
+```
+
+4. Setup ansible, run the playbook.
+```
+cd meshdb/infra/ansible
+ansible-galaxy collection install cloud.terraform
+ansible-playbook -i inventory.yaml meshdb.yaml
+```
+
+5. Install the `meshdb-cluster` chart.
+
+```
+cd meshdb/infra/helm/meshdb-cluster
+# Modify values.yaml to meet your needs
+helm template . -f values.yaml > meshdb-cluster.yaml
+kubectl apply --kubeconfig='../../tf/k3s.yaml' -f meshdb-cluster.yaml
+# Watch everything come up
+kubectl get all --kubeconfig='../../tf/k3s.yaml' --namespace longhorn-system
+```
+
+5. Create and update values + secrets in `values.yaml` and `secret.values.yaml`
+
+```
+cd meshdb/infra/helm/meshdb/
+cp example.secret.values.yaml secret.values.yaml
+cp example.values.yaml values.yaml
+nano secret.values.yaml
+nano values.yaml
+```
+
+6. Install the `meshdb` chart.
+
+```
+cd meshdb/infra/helm/meshdb
+helm template . -f secret.values.yaml -f values.yaml > meshdb.yaml
+kubectl apply --kubeconfig='../../tf/k3s.yaml' -f meshdb.yaml
+# Watch everything come up
+kubectl get all --kubeconfig='../../tf/k3s.yaml' --namespace meshdbdev3
+```
+
+7. If you need a superuser: `kubectl exec -it -n meshdbdev3 service/meshdb-meshweb bash` and `python manage.py createsuperuser`

infra/ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+host_key_checking = False
+
+[ssh_connection]
+ssh_args = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no'

infra/ansible/inventory.yaml

@@ -0,0 +1,3 @@
+---
+plugin: cloud.terraform.terraform_provider
+project_path: "../tf"

infra/ansible/meshdb.yaml

@@ -0,0 +1,11 @@
+- hosts: mgrs:workers 
+  roles:
+    - role: meshdb-k8s-node
+
+- hosts: mgrs 
+  roles:
+    - role: meshdb-mgr
+
+- hosts: lb
+  roles:
+    - role: k8s-lb