From 322b6d14fce5a8004f212d271f29c732a0fcd7a9 Mon Sep 17 00:00:00 2001
From: msantos <msantos@wikimedia.org>
Date: Sat, 24 Aug 2024 20:07:16 +0200
Subject: [PATCH] Migrate the config.yaml generation to PKL

---
 .github/workflows/aws.yml                     | 26 ++++----------
 deployment/config/config-file/development.pkl | 16 +++++++++
 .../config/config-file/modules/agencia.pkl    |  1 +
 .../config/config-file/modules/aws/s3.pkl     |  3 ++
 .../config-file/modules/database/mongodb.pkl  |  8 +++++
 .../config-file/modules/feature_flag.pkl      |  3 ++
 .../config/config-file/modules/main.pkl       | 34 +++++++++++++++++++
 .../config/config-file/modules/novu.pkl       |  2 ++
 .../config/config-file/modules/openai.pkl     |  2 ++
 deployment/config/config-file/modules/ory.pkl |  5 +++
 .../config-file/modules/service/base.pkl      |  4 +++
 .../config-file/modules/service/confBase.pkl  | 31 +++++++++++++++++
 .../config/config-file/modules/zenvia.pkl     |  2 ++
 deployment/config/config-file/production.pkl  | 16 +++++++++
 14 files changed, 133 insertions(+), 20 deletions(-)
 create mode 100644 deployment/config/config-file/development.pkl
 create mode 100644 deployment/config/config-file/modules/agencia.pkl
 create mode 100644 deployment/config/config-file/modules/aws/s3.pkl
 create mode 100644 deployment/config/config-file/modules/database/mongodb.pkl
 create mode 100644 deployment/config/config-file/modules/feature_flag.pkl
 create mode 100644 deployment/config/config-file/modules/main.pkl
 create mode 100644 deployment/config/config-file/modules/novu.pkl
 create mode 100644 deployment/config/config-file/modules/openai.pkl
 create mode 100644 deployment/config/config-file/modules/ory.pkl
 create mode 100644 deployment/config/config-file/modules/service/base.pkl
 create mode 100644 deployment/config/config-file/modules/service/confBase.pkl
 create mode 100644 deployment/config/config-file/modules/zenvia.pkl
 create mode 100644 deployment/config/config-file/production.pkl

diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml
index 968d5f87f..3623bab86 100644
--- a/.github/workflows/aws.yml
+++ b/.github/workflows/aws.yml
@@ -57,6 +57,7 @@ jobs:
           sed -i '21{s/test.//}' deployment/app.yml
           sed -i '11{s/testws./ws./}' deployment/websocket.yml
           sed -i '11{s/test.//}' config.seed.example.yaml
+          echo "ENV=production" >> $GITHUB_ENV
           echo "ENVIRONMENT=production" >> $GITHUB_ENV
           echo "UMAMI_SITE_ID=${{ secrets.PRODUCTION_UMAMI_SITE_ID }}" >> $GITHUB_ENV
           echo "NEXT_PUBLIC_UMAMI_SITE_ID=${{ secrets.PRODUCTION_UMAMI_SITE_ID }}" >> $GITHUB_ENV
@@ -90,29 +91,14 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
 
+      - name: Install pkl
+        uses: pkl-community/setup-pkl@v0
+        with:
+          pkl-version: 0.25.2
       # Setting up config.yaml based on environment
       - name: Set config.yaml
-        env:
-          RECAPTCHA_SECRET: ${{ secrets.RECAPTCHA_SECRETKEY }}
         run: |
-          sed -i "s%ZENVIA_API_URL%$ZENVIA_API_URL%g" config.$ENVIRONMENT.yaml
-          sed -i "s%ZENVIA_API_TOKEN%$ZENVIA_API_TOKEN%g" config.$ENVIRONMENT.yaml
-          sed -i "s/ENV/$ENVIRONMENT/g" config.$ENVIRONMENT.yaml
-          sed -i "s%RECAPTCHA_SECRET%$RECAPTCHA_SECRET%g" config.$ENVIRONMENT.yaml
-          sed -i "s%MONGODB_URI%$MONGODB_URI%g" config.$ENVIRONMENT.yaml
-          sed -i "s%ORY_SDK_URL%$ORY_SDK_URL%g" config.$ENVIRONMENT.yaml
-          sed -i "s%GITLAB_FEATURE_FLAG_URL%$GITLAB_FEATURE_FLAG_URL%g" config.$ENVIRONMENT.yaml
-          sed -i "s%GITLAB_FEATURE_FLAG_INSTANCE_ID%$GITLAB_FEATURE_FLAG_INSTANCE_ID%g" config.$ENVIRONMENT.yaml
-          sed -i "s/ORY_ACCESS_TOKEN/$ORY_ACCESS_TOKEN/g" config.$ENVIRONMENT.yaml
-          sed -i "s/ALETHEIA_SCHEMA_ID/$ALETHEIA_SCHEMA_ID/g" config.$ENVIRONMENT.yaml
-          sed -i "s%AWS_SDK_BUCKET%$AWS_SDK_BUCKET%g" config.$ENVIRONMENT.yaml
-          sed -i "s%AWS_ACCESS_KEY_ID%$AWS_ACCESS_KEY_ID%g" config.$ENVIRONMENT.yaml
-          sed -i "s%AWS_SECRET_ACCESS_KEY%$AWS_SECRET_ACCESS_KEY%g" config.$ENVIRONMENT.yaml
-          sed -i "s%NOVU_API_KEY%$NOVU_API_KEY%g" config.$ENVIRONMENT.yaml
-          sed -i "s%NOVU_APPLICATION_IDENTIFIER%$NOVU_APPLICATION_IDENTIFIER%g" config.$ENVIRONMENT.yaml
-          sed -i "s%AGENTS_API_URL%$AGENTS_API_URL%g" config.$ENVIRONMENT.yaml
-          sed -i "s%OPENAI_API_KEY%$OPENAI_API_KEY%g" config.$ENVIRONMENT.yaml
-          sed -i "s%AGENCIA_ACCESS_TOKEN%$AGENCIA_ACCESS_TOKEN%g" config.$ENVIRONMENT.yaml
+          pkl eval -f yaml ./deployment/config/config-file/$ENVIRONMENT.pkl > config.$ENVIRONMENT.yaml
 
       - name: Set migrate-mongo-config.ts
         run: |
diff --git a/deployment/config/config-file/development.pkl b/deployment/config/config-file/development.pkl
new file mode 100644
index 000000000..45fe80914
--- /dev/null
+++ b/deployment/config/config-file/development.pkl
@@ -0,0 +1,16 @@
+amends "./modules/main.pkl"
+import "./modules/database/mongodb.pkl"
+appBaseUrl = "aletheiafact.org"
+
+var = new {
+    name = "aletheia-development"
+    conf {
+        cors = "*"
+        recaptcha_sitekey = "6Lc2BtYUAAAAAOUBI-9r1sDJUIfG2nt6C43noOXh"
+        websocketUrl = "wss://testws.\(appBaseUrl) "
+        baseUrl = "https://test.\(appBaseUrl)"
+        db = (mongodb) {
+            atlas = true
+        }
+    }
+}
diff --git a/deployment/config/config-file/modules/agencia.pkl b/deployment/config/config-file/modules/agencia.pkl
new file mode 100644
index 000000000..510c4fa1a
--- /dev/null
+++ b/deployment/config/config-file/modules/agencia.pkl
@@ -0,0 +1 @@
+access_token = read("env:AGENCIA_ACCESS_TOKEN")
diff --git a/deployment/config/config-file/modules/aws/s3.pkl b/deployment/config/config-file/modules/aws/s3.pkl
new file mode 100644
index 000000000..b19e7599f
--- /dev/null
+++ b/deployment/config/config-file/modules/aws/s3.pkl
@@ -0,0 +1,3 @@
+bucket = read("env:AWS_SDK_BUCKET")
+accessKeyId = read("env:AWS_ACCESS_KEY_ID")
+secretAccessKey = read("env:AWS_SECRET_ACCESS_KEY")
diff --git a/deployment/config/config-file/modules/database/mongodb.pkl b/deployment/config/config-file/modules/database/mongodb.pkl
new file mode 100644
index 000000000..121337739
--- /dev/null
+++ b/deployment/config/config-file/modules/database/mongodb.pkl
@@ -0,0 +1,8 @@
+atlas = false
+connection_uri = read("env:MONGODB_URI")
+options = new {
+    useUnifiedTopology = true
+    useNewUrlParser = true
+    retryWrites =  true
+    w =  "majority"
+}
diff --git a/deployment/config/config-file/modules/feature_flag.pkl b/deployment/config/config-file/modules/feature_flag.pkl
new file mode 100644
index 000000000..710329557
--- /dev/null
+++ b/deployment/config/config-file/modules/feature_flag.pkl
@@ -0,0 +1,3 @@
+url = read("env:GITLAB_FEATURE_FLAG_URL")
+appName = read("env:ENV")
+instanceId = read("env:GITLAB_FEATURE_FLAG_INSTANCE_ID")
diff --git a/deployment/config/config-file/modules/main.pkl b/deployment/config/config-file/modules/main.pkl
new file mode 100644
index 000000000..f2e0961d1
--- /dev/null
+++ b/deployment/config/config-file/modules/main.pkl
@@ -0,0 +1,34 @@
+import "./database/mongodb.pkl"
+import "./service/confBase.pkl"
+import "./ory.pkl" as oryConfig
+
+
+hidden appBaseUrl: String
+
+hidden var = new {
+    name = "service-aletheia"
+    conf = new {
+        port = 3000
+        recaptcha_sitekey = read("env:RECAPTCHA_SITEKEY")
+        websocketUrl = read("env:WEBSOCKET_URL")
+        baseUrl = read("env:BASE_URL")
+        automatedFactCheckingAPIUrl = read("env:AGENTS_API_URL")
+        recaptcha_secret = read("env:RECAPTCHA_SECRET")
+        throttle = new confBase.Throttle {
+            ttl = 60
+            limit = 100
+        }
+        ory = (oryConfig) {
+            admin_endpoint = "admin"
+        }
+    }
+}
+
+services = new Listing {
+    new {
+        name = var.name
+        conf = (confBase) {
+            ...var.conf
+        }
+    }
+}
diff --git a/deployment/config/config-file/modules/novu.pkl b/deployment/config/config-file/modules/novu.pkl
new file mode 100644
index 000000000..171da3fac
--- /dev/null
+++ b/deployment/config/config-file/modules/novu.pkl
@@ -0,0 +1,2 @@
+api_key = read("env:NOVU_API_KEY")
+application_identifier = read("env:NOVU_APPLICATION_IDENTIFIER")
diff --git a/deployment/config/config-file/modules/openai.pkl b/deployment/config/config-file/modules/openai.pkl
new file mode 100644
index 000000000..3e4c05d6c
--- /dev/null
+++ b/deployment/config/config-file/modules/openai.pkl
@@ -0,0 +1,2 @@
+api_key = read("env:OPENAI_API_KEY")
+
diff --git a/deployment/config/config-file/modules/ory.pkl b/deployment/config/config-file/modules/ory.pkl
new file mode 100644
index 000000000..b804b978f
--- /dev/null
+++ b/deployment/config/config-file/modules/ory.pkl
@@ -0,0 +1,5 @@
+url = read("env:ORY_SDK_URL")
+admin_url = read("env:ORY_SDK_URL")
+admin_endpoint: String
+access_token = read("env:ORY_ACCESS_TOKEN")
+schema_id = read("env:ALETHEIA_SCHEMA_ID")
diff --git a/deployment/config/config-file/modules/service/base.pkl b/deployment/config/config-file/modules/service/base.pkl
new file mode 100644
index 000000000..e30b8cc4f
--- /dev/null
+++ b/deployment/config/config-file/modules/service/base.pkl
@@ -0,0 +1,4 @@
+class BaseService {
+    name: String
+    conf: Map<String, Value>
+}
diff --git a/deployment/config/config-file/modules/service/confBase.pkl b/deployment/config/config-file/modules/service/confBase.pkl
new file mode 100644
index 000000000..7cd1811a4
--- /dev/null
+++ b/deployment/config/config-file/modules/service/confBase.pkl
@@ -0,0 +1,31 @@
+import "../database/mongodb.pkl" as mongoDbConfig
+import "../ory.pkl" as oryConfig
+import "../feature_flag.pkl" as featureFlagConfig
+import "../aws/s3.pkl" as s3Config
+import "../novu.pkl" as novuConfig
+import "../openai.pkl" as openaiConfig
+import "../zenvia.pkl" as zenviaConfig
+import "../agencia.pkl" as agenciaConfig
+
+class Throttle {
+    ttl: Number
+    limit: Number
+}
+
+port: Number
+cors: String
+websocketUrl: String
+baseUrl: String
+automatedFactCheckingAPIUrl: String
+recaptcha_secret: String
+recaptcha_sitekey: String
+throttle: Throttle
+authentication_type = "ory"
+db = (mongoDbConfig) {}
+ory = (oryConfig) {}
+feature_flag = (featureFlagConfig) {}
+aws = (s3Config) {}
+novu = (novuConfig) {}
+openai = (openaiConfig) {}
+zenvia = (zenviaConfig) {}
+agencia = (agenciaConfig) {}
diff --git a/deployment/config/config-file/modules/zenvia.pkl b/deployment/config/config-file/modules/zenvia.pkl
new file mode 100644
index 000000000..79c13ef73
--- /dev/null
+++ b/deployment/config/config-file/modules/zenvia.pkl
@@ -0,0 +1,2 @@
+api_url = read("env:ZENVIA_API_URL")
+api_token = read("env:ZENVIA_API_TOKEN")
diff --git a/deployment/config/config-file/production.pkl b/deployment/config/config-file/production.pkl
new file mode 100644
index 000000000..33c525d20
--- /dev/null
+++ b/deployment/config/config-file/production.pkl
@@ -0,0 +1,16 @@
+amends "./modules/main.pkl"
+import "./modules/database/mongodb.pkl"
+appBaseUrl = "aletheiafact.org"
+
+var = new {
+    name = "aletheia-production"
+    conf {
+        cors = "*"
+        recaptcha_sitekey = "6Lc2BtYUAAAAAOUBI-9r1sDJUIfG2nt6C43noOXh"
+        websocketUrl = "wss://ws.\(appBaseUrl) "
+        baseUrl = "https://\(appBaseUrl)"
+        db = (mongodb) {
+            atlas = true
+        }
+    }
+}