Powering Automatic Authorization with live traffic demo

.gitignore

@@ -16,4 +16,7 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 
-custom-k8s-metrics-demo/main
+custom-k8s-metrics-demo/main
+
+automatic-authz-envoy-opa/__pycache__
+**/__pycache__/

automatic-authz-envoy-opa/README.md

@@ -0,0 +1,21 @@
+## Automatic AuthZ through with Envoy and OPA
+
+This demo showcases how to configure Envoy and OPA to automatically authorize requests based on traffic patterns seen from Pixie's protocol tracing data.
+
+### Prerequisites
+- `kubectl` installed
+- K8s cluster with Pixie installed and `px` cli installed. Follow the instructions [here](https://docs.px.dev/installing-pixie/install-schemes/cli/).
+- AWS account with an S3 bucket and access key and secret key with permissions to read and write to the bucket.
+
+
+### Deployment
+1. Update OTel Collector configuration with S3 bucket details and add credential files used by `kustomize`
+- Add the AWS access key and secret key to the envoy-opa/otel-collector/{AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY} files and to your `~/.aws/credentials` file. Note: Make sure your credentials files 
+- S3 bucket details modified in `envoy-opa/otel-collector/otel-collector-config.yaml`. Search for the `PLACEHOLDER` comment for the locations to modify.
+
+2. Clone the repo and change the OPA bundle location in envoy-opa/kustomization.yaml to your fork
+
+3. Deploy the application
+```bash
+$ ./deploy-spire-and-demo-applications.sh
+```

automatic-authz-envoy-opa/deploy-spire-and-demo-applications.sh

@@ -0,0 +1,145 @@
+#/bin/bash
+
+# This was adapted from the original script in the SPIRE repository
+# https://github.com/spiffe/spire-tutorials/blob/d27c579eb4f4e26f36f60373446c42c4ebd1e3da/k8s/envoy-x509/scripts/pre-set-env.sh
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PREREQS_DIR="$DIR/prereqs"
+OPA_DIR="$DIR/envoy-opa"
+
+bb=$(tput bold) || true
+nn=$(tput sgr0) || true
+red=$(tput setaf 1) || true
+green=$(tput setaf 2) || true
+
+register() {
+    kubectl exec -n spire spire-server-0 -c spire-server -- /opt/spire/bin/spire-server entry create $@
+}
+
+wait_for_agent() {
+    for ((i=0;i<120;i++)); do
+        if ! kubectl -nspire rollout status statefulset/spire-server; then
+            sleep 1
+            continue
+        fi
+        if ! kubectl -nspire rollout status daemonset/spire-agent; then
+            sleep 1
+            continue
+        fi
+        if ! kubectl -nspire logs statefulset/spire-server -c spire-server | grep -e "$LOGLINE" ; then
+            sleep 1
+            continue
+        fi
+        echo "${bold}SPIRE Agent ready.${nn}"
+        RUNNING=1
+        break
+    done
+    if [ ! -n "${RUNNING}" ]; then
+        echo "${red}Timed out waiting for SPIRE Agent to be running.${nn}"
+        exit 1
+    fi
+}
+
+restart_deployments() {
+    kubectl scale deployment backend --replicas=0
+    kubectl scale deployment backend --replicas=1
+
+    kubectl scale deployment frontend --replicas=0
+    kubectl scale deployment frontend --replicas=1
+
+    kubectl scale deployment frontend-2 --replicas=0
+    kubectl scale deployment frontend-2 --replicas=1
+}
+
+wait_for_envoy() {
+    # wait until deployments are completed and Envoy is ready
+    LOGLINE="DNS hosts have changed for backend-envoy"
+
+    for ((i=0;i<30;i++)); do
+        if ! kubectl rollout status deployment/backend; then
+            sleep 1
+            continue
+        fi
+        if ! kubectl rollout status deployment/frontend; then
+            sleep 1
+            continue
+        fi
+        if ! kubectl rollout status deployment/frontend-2; then
+            sleep 1
+            continue
+        fi
+        if ! kubectl logs --tail=300 --selector=app=frontend -c envoy | grep -qe "${LOGLINE}" ; then
+            sleep 5
+            echo "Waiting until Envoy is ready..."
+            continue
+        fi
+        echo "Workloads ready."
+        WK_READY=1
+        break
+    done
+    if [ -z "${WK_READY}" ]; then
+        echo "${red}Timed out waiting for workloads to be ready.${nn}"
+        exit 1
+    fi
+}
+
+echo "${bb}Creates all the resources needed to the SPIRE Server and SPIRE Agent to be available in the cluster.${nn}"
+kubectl apply -k ${PREREQS_DIR} > /dev/null
+
+echo "${bb}Waiting until SPIRE Agent is running${nn}"
+wait_for_agent
+
+echo "${bb}Creates spire agent registration entries.${nn}"
+kubectl exec -n spire spire-server-0 -- \
+    /opt/spire/bin/spire-server entry create \
+    -node  \
+    -spiffeID spiffe://example.org/ns/spire/sa/spire-agent \
+    -selector k8s_sat:cluster:demo-cluster \
+    -selector k8s_sat:agent_ns:spire \
+    -selector k8s_sat:agent_sa:spire-agent
+
+
+echo "${green}SPIRE agent resources creation completed.${nn}"
+
+echo "${bb}Creating registration entry for the backend - envoy...${nn}"
+register \
+    -parentID spiffe://example.org/ns/spire/sa/spire-agent \
+    -spiffeID spiffe://example.org/ns/default/sa/default/backend \
+    -selector k8s:ns:default \
+    -selector k8s:sa:default \
+    -selector k8s:pod-label:app:backend \
+    -selector k8s:container-name:envoy
+
+echo "${bb}Creating registration entry for the frontend - envoy...${nn}"
+register \
+    -parentID spiffe://example.org/ns/spire/sa/spire-agent \
+    -spiffeID spiffe://example.org/ns/default/sa/default/frontend \
+    -selector k8s:ns:default \
+    -selector k8s:sa:default \
+    -selector k8s:pod-label:app:frontend \
+    -selector k8s:container-name:envoy
+
+echo "${bb}Creating registration entry for the frontend - envoy...${nn}"
+register \
+    -parentID spiffe://example.org/ns/spire/sa/spire-agent \
+    -spiffeID spiffe://example.org/ns/default/sa/default/frontend-2 \
+    -selector k8s:ns:default \
+    -selector k8s:sa:default \
+    -selector k8s:pod-label:app:frontend-2 \
+    -selector k8s:container-name:envoy
+
+echo "${bb}Listing created registration entries...${nn}"
+kubectl exec -n spire spire-server-0 -- /opt/spire/bin/spire-server entry show
+
+echo "${bb}Applying SPIRE Envoy with OPA configuration...${nn}"
+kubectl apply -k $OPA_DIR/ > /dev/null
+
+# Restarts all deployments to pickup the new configurations
+restart_deployments > /dev/null
+
+echo "${bb}Waiting until deployments and Envoy are ready...${nn}"
+wait_for_envoy > /dev/null
+
+echo "${bb}X.509 Environment creation completed.${nn}"

automatic-authz-envoy-opa/envoy-opa/kustomization.yaml

@@ -0,0 +1,37 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://github.com/spiffe/spire-tutorials/k8s/envoy-opa/k8s?ref=d27c579eb4f4e26f36f60373446c42c4ebd1e3da
+  - otel-collector/
+
+patches:
+    # Remove rego file from Configmap and add new configuration that loads the OPA policy
+    # from a bundle
+  - target:
+      kind: ConfigMap
+      name: backend-opa-policy-config
+    patch: |-
+      - op: remove
+        path: /data/opa-policy.rego
+      - op: replace
+        path: /data/opa-config.yaml
+        value: |
+          decision_logs:
+            console: true
+          plugins:
+            envoy_ext_authz_grpc:
+              addr: :8182
+              query: data.envoy.authz.allow
+          services:
+            - name: github
+              url: https://github.com/ddelnano/pixie-demos/raw/refs/heads/ddelnano/kubecon-2024-authz/automatic-authz-envoy-opa
+          bundles:
+            authz:
+              service: github
+              resource: opa/bundle.tar.gz
+              persist: true
+              polling:
+                min_delay_seconds: 10
+                max_delay_seconds: 20
+  - path: patches/envoy-backend.yaml

automatic-authz-envoy-opa/envoy-opa/otel-collector/.gitignore

@@ -0,0 +1,2 @@
+AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY

automatic-authz-envoy-opa/envoy-opa/otel-collector/collector.yaml

@@ -0,0 +1,144 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-conf
+  labels:
+    app: opentelemetry
+    component: otel-collector-conf
+data:
+  otel-collector-config: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+    processors:
+      batch:
+      memory_limiter:
+        # 80% of maximum memory up to 2G
+        limit_mib: 1500
+        # 25% of limit up to 2G
+        spike_limit_mib: 512
+        check_interval: 5s
+    extensions:
+      zpages: {}
+    exporters:
+      # PLACEHOLDER: Uncomment and replace with your AWS S3 bucket details
+      # awss3:
+      #   s3uploader:
+      #     region: 'us-west-2'
+      #     s3_bucket: 'kubecon-2024-automatic-authorization'
+      #     s3_prefix: 'otel-export'
+      #     s3_partition: 'minute'
+      file:
+        path: /file-exporter/data.json
+      debug:
+        verbosity: detailed
+        sampling_initial: 5
+        sampling_thereafter: 200
+    service:
+      extensions: [zpages]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          # PLACEHOLDER: Replace the following line with the commented one
+          # to enable the AWS S3 exporter
+          # exporters: [debug, file]
+          exporters: [debug, file, awss3]
+        metrics:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [debug, file]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  ports:
+  - name: otlp-grpc # Default endpoint for OpenTelemetry gRPC receiver.
+    port: 4317
+    protocol: TCP
+    targetPort: 4317
+  - name: otlp-http # Default endpoint for OpenTelemetry HTTP receiver.
+    port: 4318
+    protocol: TCP
+    targetPort: 4318
+  - name: metrics # Default endpoint for querying metrics.
+    port: 8888
+  selector:
+    component: otel-collector
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  selector:
+    matchLabels:
+      app: opentelemetry
+      component: otel-collector
+  minReadySeconds: 5
+  progressDeadlineSeconds: 120
+  replicas: 1 #TODO - adjust this to your own requirements
+  template:
+    metadata:
+      labels:
+        app: opentelemetry
+        component: otel-collector
+    spec:
+      containers:
+      - command:
+          - "/otelcol-contrib"
+          - "--config=/conf/otel-collector-config.yaml"
+        image: otel/opentelemetry-collector-contrib:0.110.0
+        name: otel-collector
+        env:
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: otel-exporter-aws-credentials
+              key: AWS_ACCESS_KEY_ID
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: otel-exporter-aws-credentials
+              key: AWS_SECRET_ACCESS_KEY
+        resources:
+          limits:
+            cpu: 1
+            memory: 2Gi
+          requests:
+            cpu: 200m
+            memory: 400Mi
+        ports:
+        - containerPort: 55679 # Default endpoint for ZPages.
+        - containerPort: 4317 # Default endpoint for OpenTelemetry receiver.
+        - containerPort: 14250 # Default endpoint for Jaeger gRPC receiver.
+        - containerPort: 14268 # Default endpoint for Jaeger HTTP receiver.
+        - containerPort: 9411 # Default endpoint for Zipkin receiver.
+        - containerPort: 8888  # Default endpoint for querying metrics.
+        volumeMounts:
+        - name: otel-collector-config-vol
+          mountPath: /conf
+        - name: file-exporter
+          mountPath: /file-exporter
+      volumes:
+        - configMap:
+            name: otel-collector-conf
+            items:
+              - key: otel-collector-config
+                path: otel-collector-config.yaml
+          name: otel-collector-config-vol
+        - name: file-exporter
+          emptyDir:
+            sizeLimit: 1Gi

automatic-authz-envoy-opa/envoy-opa/otel-collector/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+secretGenerator:
+  - name: otel-exporter-aws-credentials
+    files:
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY
+resources:
+  - collector.yaml

automatic-authz-envoy-opa/envoy-opa/patches/envoy-backend.yaml

@@ -0,0 +1,18 @@
+$patch: merge
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend
+  labels:
+    app: backend
+spec:
+  template:
+    spec:
+      containers:
+        - name: opa
+          # The existing OPA image is too old for rego.v1 imports
+          image: openpolicyagent/opa:0.59.0-envoy
+          args:
+          - run
+          - --server
+          - --config-file=/run/opa/opa-config.yaml