Skip to content

Commit

Permalink
Adding Kubernetes Application resource manifests for microservices …
Browse files Browse the repository at this point in the history
…| Disabling `dorny paths filter` in Github Actions | Update Readme.md
  • Loading branch information
Archisman-Mridha committed Jan 28, 2024
1 parent e9d1acc commit 5516c30
Show file tree
Hide file tree
Showing 17 changed files with 396 additions and 63 deletions.
65 changes: 31 additions & 34 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ name: Main
on:
push:
branches: [main]
pull_request:
branches: [dev, main]
types: [opened, edited]

## Adding this allows us to trigger this workflow manually (Just for debugging purposes).
workflow_dispatch: {}
Expand Down Expand Up @@ -131,8 +128,8 @@ jobs:
- go.work.sum
- name: application-controller
path: backend/gateway
manifest: kubernetes/base/microservices/application-controller.application.yaml
path: kubernetes/operators/application
manifest: kubernetes/base/application-controller/deployment.yaml
path_filters: |
changed:
- kubernetes/operators/application/**
Expand All @@ -144,31 +141,31 @@ jobs:

## If sourcecode of the microservice has changed, only then we will rebuild, push, sign
## and scan the container image.
- name: Detect sourcecode change
uses: dorny/paths-filter@v2
id: path-filter
with:
base: ${{ github.ref }}
filters: ${{ matrix.microservice.path_filters }}
## - name: Detect sourcecode change
## uses: dorny/paths-filter@v2
## id: path-filter
## with:
## base: ${{ github.ref }}
## filters: ${{ matrix.microservice.path_filters }}

- name: Set up QEMU
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: docker/setup-buildx-action@v2

- name: Login to GitHub Container Registry
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: docker/[email protected]
with:
registry: ghcr.io
username: archisman-mridha
password: ${{ secrets.GITHUB_TOKEN }}

- name: Restore cached Cargo dependencies (if exists)
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: actions/cache/restore@v3
with:
path: |
Expand All @@ -177,7 +174,7 @@ jobs:
key: ${{ runner.os }}-cargo-${{ matrix.microservice.name }}-${{ hashFiles('**/Cargo.lock') }}

- name: Build and push AMD64 container image
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: docker/build-push-action@v4
with:
context: .
Expand All @@ -194,7 +191,7 @@ jobs:
cache-to: type=gha,mode=max

- name: Cache Cargo dependencies
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: actions/cache@v3
with:
path: |
Expand All @@ -203,20 +200,20 @@ jobs:
key: ${{ runner.os }}-cargo-${{ matrix.microservice.name }}-${{ hashFiles('**/Cargo.lock') }}

- name: Remove cached folders from local machine
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
run: |
rm -rf /usr/local/cargo/registry/ target/
## Cosign is a command line utility that can sign and verify software artifact, such as
## container images and blobs.
- name: Install Cosign
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: sigstore/[email protected]
with:
cosign-release: v2.2.1

- name: Sign the published container image
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
env:
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
run: |
Expand All @@ -228,11 +225,11 @@ jobs:
ghcr.io/archisman-mridha/instagram-clone-${{ matrix.microservice.name }}:${{ github.sha }} -y
- name: Create outputs directory
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
run: mkdir -p ./outputs/trivy

- name: Scan container image for vulnerabilities
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/archisman-mridha/instagram-clone-${{ matrix.microservice.name }}:${{ github.sha }}
Expand All @@ -243,23 +240,23 @@ jobs:
output: ./outputs/trivy/${{ matrix.microservice.name }}.container-image-scan-result.sarif

- name: Upload the scan result as Github artifact
if: steps.path-filter.outputs.changed == 'true'
## if: steps.path-filter.outputs.changed == 'true'
uses: actions/upload-artifact@v3
with:
name: trivy.${{ matrix.microservice.name }}-microservice.container-image-scan-result.sarif
path: ./outputs/trivy/${{ matrix.microservice.name }}.container-image-scan-result.sarif

## - name: Update container image tag in Kubernetes manifests
## if: steps.path-filter.outputs.changed == 'true'
## run: |
## git config --global user.name "Archisman-Mridha"
## git config --global user.email "[email protected]"
## git config --global pull.rebase false
- name: Update container image tag in Kubernetes manifests
## if: steps.path-filter.outputs.changed == 'true'
run: |
git config --global user.name "Archisman-Mridha"
git config --global user.email "[email protected]"
git config --global pull.rebase false
## git pull origin main
git pull origin main -f
## sed -i 's/instagram-clone-\(.*\):[[:alnum:]]\+/instagram-clone-\1:${{ github.sha }}/g' ${{ matrix.microservice.manifest }}
sed -i 's/instagram-clone-\(.*\):[[:alnum:]]\+/instagram-clone-\1:${{ github.sha }}/g' ${{ matrix.microservice.manifest }}
## git add .
## git commit -m "🤖 Update container image tag for ${{ matrix.microservice.name }} to ${{ github.sha }}"
## git push --set-upstream origin main
git add .
git commit -m "🤖 Update container image tag for ${{ matrix.microservice.name }} to ${{ github.sha }}"
git push --set-upstream origin main
8 changes: 5 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Instagram Clone

![Main Workflow](https://github.com/Archisman-Mridha/instagram-clone/actions/workflows/main.yaml/badge.svg)

Welcome to the `Instagram Clone` project, which aims to replicate the core functionalities of the popular social media platform, **Instagram**, while demonstrating :

> How to build and deploy fault tolerant distributed systems.
Expand All @@ -16,9 +18,9 @@ If you encounter any bugs, have feature requests, or just want to discuss about
## References

- [Microservices Patterns](https://www.oreilly.com/library/view/microservices-patterns/9781617294549/) book by Chris Richardson
- [Jordan has no life](https://www.youtube.com/@jordanhasnolife5163)'s [Youtube video](https://www.youtube.com/watch?v=S2y9_XYOZsg&pp=ygUqam9yZGFuIGhhcyBubyBsaWZlIGluc3RhZ3JhbSBzeXN0ZW0gZGVzaWdu) on how to design a social media platform like Instagram
- [Jordan has no life](https://www.youtube.com/@jordanhasnolife5163)'s Youtube video on [how to design a social media platform like Instagram](https://www.youtube.com/watch?v=S2y9_XYOZsg&pp=ygUqam9yZGFuIGhhcyBubyBsaWZlIGluc3RhZ3JhbSBzeXN0ZW0gZGVzaWdu)
- The [Rust Book](https://doc.rust-lang.org/book/)
- [Crust of Rust](https://youtube.com/playlist?list=PLqbS7AVVErFiWDOAVrPt7aYmnuuOLYvOa&si=-Y5Byig03EhhX0pi) Youtube playlist by [Jon Gjengset](https://www.youtube.com/@jonhoo)
- Youtube channels - [Mario Carrion](https://www.youtube.com/@MarioCarrion/playlists) | [Viktor Farcic](https://www.youtube.com/@DevOpsToolkit)
- [Vivek Singh](https://www.youtube.com/@viveksinghggits)'s [Youtube playlist](https://www.youtube.com/playlist?list=PLh4KH3LtJvRTtFWz1WGlyDa7cKjj2Sns0) on how to write a custom Kubernetes controller
- [Heiko's Blogs](https://heikoseeberger.de/tags/opentelemetry/) on how to instrument Rust codebase for Distrbuted Tracing
- [Vivek Singh](https://www.youtube.com/@viveksinghggits)'s Youtube playlist on [how to write a custom Kubernetes controller](https://www.youtube.com/playlist?list=PLh4KH3LtJvRTtFWz1WGlyDa7cKjj2Sns0)
- [Heiko's Blogs](https://heikoseeberger.de) on [how to instrument Rust codebase for Distrbuted Tracing](https://heikoseeberger.de/tags/opentelemetry/)
88 changes: 64 additions & 24 deletions compose.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
version: '3'
version: "3"

services:

postgres:
container_name: postgres
image: postgres:alpine
Expand All @@ -13,9 +12,28 @@ services:
ports:
- 5432:5432
## 'wal_level = logical' is the highest level of WAL logging.
command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4", "-c", "max_wal_senders=4"]
command:
[
"postgres",
"-c",
"wal_level=logical",
"-c",
"max_replication_slots=4",
"-c",
"max_wal_senders=4",
]
healthcheck:
test: ["CMD", "psql", "-U", "default", "-d", "instagram_clone", "-c", "SELECT 1"]
test:
[
"CMD",
"psql",
"-U",
"default",
"-d",
"instagram_clone",
"-c",
"SELECT 1",
]
interval: 10s
timeout: 5s
retries: 5
Expand All @@ -25,7 +43,14 @@ services:
image: migrate/migrate:latest
volumes:
- ./backend/sql/schema.sql:/migrations/000001_init.up.sql:ro
command: ["-path", "/migrations", "-database", "postgres://default:pass@postgres:5432/instagram_clone?sslmode=disable", "up"]
command:
[
"-path",
"/migrations",
"-database",
"postgres://default:pass@postgres:5432/instagram_clone?sslmode=disable",
"up",
]
depends_on:
postgres:
condition: service_healthy
Expand Down Expand Up @@ -65,15 +90,21 @@ services:
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
healthcheck:
test: ["CMD", "kafka-topics.sh", "--bootstrap-server=localhost:9092", "--list"]
test:
[
"CMD",
"kafka-topics.sh",
"--bootstrap-server=localhost:9092",
"--list",
]
start_period: 15s
interval: 10s

kafka-migrate:
container_name: kafka-migrate
image: bitnami/kafka
working_dir: /opt/bitnami/kafka/bin
entrypoint: [ "/bin/sh", "-c" ]
entrypoint: ["/bin/sh", "-c"]
depends_on:
kafka:
condition: service_healthy
Expand Down Expand Up @@ -133,7 +164,16 @@ services:
VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
ENABLE_DEBEZIUM_SCRIPTING: "true"
healthcheck:
test: ["CMD", "curl", "--silent", "--fail", "-X", "GET", "http://localhost:8083/connectors"]
test:
[
"CMD",
"curl",
"--silent",
"--fail",
"-X",
"GET",
"http://localhost:8083/connectors",
]
start_period: 10s
interval: 10s
timeout: 5s
Expand Down Expand Up @@ -231,28 +271,28 @@ services:
depends_on:
- elasticsearch

## prometheus:
## container_name: prometheus
## image: prom/prometheus
## ports:
## - 9090:9090
## volumes:
## - ./prometheus.dev-config.yaml:/etc/prometheus/prometheus.yml:ro
prometheus:
container_name: prometheus
image: prom/prometheus
ports:
- 9090:9090
volumes:
- ./prometheus.dev-config.yaml:/etc/prometheus/prometheus.yml:ro

## autometrics:
## container_name: autometrics
## image: autometrics/am-proxy
## environment:
## - PROMETHEUS_URL=http://prometheus:9090
## ports:
## - 6789:6789
autometrics:
container_name: autometrics
image: autometrics/am-proxy
environment:
- PROMETHEUS_URL=http://prometheus:9090
ports:
- 6789:6789

jaeger:
container_name: jaeger
image: jaegertracing/all-in-one:latest
hostname: jaeger
ports:
- 4317:4317 ## gRPC endpoint
- 4317:4317 ## gRPC endpoint
- 16686:16686 ## UI dashboard endpoint
environment:
- COLLECTOR_OTLP_ENABLED=true
Expand Down Expand Up @@ -461,4 +501,4 @@ services:
driver: fluentd
options:
fluentd-address: localhost:24224
tag: gateway
tag: gateway
2 changes: 1 addition & 1 deletion kubernetes/base/application-controller/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ spec:
serviceAccountName: application-controller
containers:
- name: application-controller
image: ghcr.io/archisman-mridha/instagram-clone-application-controller:78cd2ead9ce9828ab6ec1265192df427eca05e90
image: ghcr.io/archisman-mridha/instagram-clone-application-controller:9574f877a5a231dbab94fa621fac97734ad5227e
resources:
requests:
memory: 128Mi
Expand Down
16 changes: 16 additions & 0 deletions kubernetes/base/microservices/feeds-microservice.application.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: instagramclone.io/v1alpha1
kind: Application
metadata:
name: feeds-microservice
namespace: microservices

spec:
image: ghcr.io/archisman-mridha/instagram-clone-feeds-microservice:9574f877a5a231dbab94fa621fac97734ad5227e
replicas:
min: 2
max: 3
resources:
cpu: 100m
memory: 100Mi
secretName: feeds-microservice
port: 4000
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: instagramclone.io/v1alpha1
kind: Application
metadata:
name: followships-microservice
namespace: microservices

spec:
image: ghcr.io/archisman-mridha/instagram-clone-followships-microservice:9574f877a5a231dbab94fa621fac97734ad5227e
replicas:
min: 2
max: 3
resources:
cpu: 100m
memory: 100Mi
secretName: followships-microservice
port: 4000
16 changes: 16 additions & 0 deletions kubernetes/base/microservices/gateway.application.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: instagramclone.io/v1alpha1
kind: Application
metadata:
name: gateway
namespace: microservices

spec:
image: ghcr.io/archisman-mridha/instagram-clone-gateway:9574f877a5a231dbab94fa621fac97734ad5227e
replicas:
min: 2
max: 3
resources:
cpu: 100m
memory: 100Mi
secretName: gateway
port: 4000
Loading

0 comments on commit 5516c30

Please sign in to comment.