From c3e4ae01e874f5f603992421c930535c74a75202 Mon Sep 17 00:00:00 2001 From: msantos Date: Sat, 24 Aug 2024 20:07:16 +0200 Subject: [PATCH] Migrate the config.yaml generation to PKL --- .github/workflows/aws.yml | 29 +++++----------- deployment/config/config-file/development.pkl | 16 +++++++++ .../config/config-file/modules/agencia.pkl | 1 + .../config/config-file/modules/aws/s3.pkl | 3 ++ .../config-file/modules/database/mongodb.pkl | 8 +++++ .../config-file/modules/feature_flag.pkl | 3 ++ .../config/config-file/modules/main.pkl | 34 +++++++++++++++++++ .../config/config-file/modules/novu.pkl | 2 ++ .../config/config-file/modules/openai.pkl | 2 ++ deployment/config/config-file/modules/ory.pkl | 5 +++ .../config-file/modules/service/base.pkl | 4 +++ .../config-file/modules/service/confBase.pkl | 31 +++++++++++++++++ .../config/config-file/modules/zenvia.pkl | 2 ++ deployment/config/config-file/production.pkl | 16 +++++++++ 14 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 deployment/config/config-file/development.pkl create mode 100644 deployment/config/config-file/modules/agencia.pkl create mode 100644 deployment/config/config-file/modules/aws/s3.pkl create mode 100644 deployment/config/config-file/modules/database/mongodb.pkl create mode 100644 deployment/config/config-file/modules/feature_flag.pkl create mode 100644 deployment/config/config-file/modules/main.pkl create mode 100644 deployment/config/config-file/modules/novu.pkl create mode 100644 deployment/config/config-file/modules/openai.pkl create mode 100644 deployment/config/config-file/modules/ory.pkl create mode 100644 deployment/config/config-file/modules/service/base.pkl create mode 100644 deployment/config/config-file/modules/service/confBase.pkl create mode 100644 deployment/config/config-file/modules/zenvia.pkl create mode 100644 deployment/config/config-file/production.pkl diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 968d5f87f..1d7e50cbc 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -21,6 +21,7 @@ env: NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} IMAGE: aletheiafact-production ENVIRONMENT: development + ENV: development UMAMI_SITE_ID: ${{ secrets.DEVELOPMENT_UMAMI_SITE_ID }} MONGODB_URI: ${{ secrets.DEVELOPMENT_MONGODB_URI }} MONGODB_NAME: ${{ secrets.DEVELOPMENT_MONGODB_NAME }} @@ -41,7 +42,7 @@ env: ZENVIA_API_URL: ${{ secrets.DEVELOPMENT_ZENVIA_API_URL }} ZENVIA_API_TOKEN: ${{ secrets.DEVELOPMENT_ZENVIA_API_URL }} AGENCIA_ACCESS_TOKEN: ${{ secrets.DEVELOPMENT_AGENCIA_ACCESS_TOKEN }} - + RECAPTCHA_SECRET: ${{ secrets.RECAPTCHA_SECRETKEY }} jobs: setup-build-publish: name: Build & Publish @@ -57,6 +58,7 @@ jobs: sed -i '21{s/test.//}' deployment/app.yml sed -i '11{s/testws./ws./}' deployment/websocket.yml sed -i '11{s/test.//}' config.seed.example.yaml + echo "ENV=production" >> $GITHUB_ENV echo "ENVIRONMENT=production" >> $GITHUB_ENV echo "UMAMI_SITE_ID=${{ secrets.PRODUCTION_UMAMI_SITE_ID }}" >> $GITHUB_ENV echo "NEXT_PUBLIC_UMAMI_SITE_ID=${{ secrets.PRODUCTION_UMAMI_SITE_ID }}" >> $GITHUB_ENV @@ -90,29 +92,14 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v1 + - name: Install pkl + uses: pkl-community/setup-pkl@v0 + with: + pkl-version: 0.25.2 # Setting up config.yaml based on environment - name: Set config.yaml - env: - RECAPTCHA_SECRET: ${{ secrets.RECAPTCHA_SECRETKEY }} run: | - sed -i "s%ZENVIA_API_URL%$ZENVIA_API_URL%g" config.$ENVIRONMENT.yaml - sed -i "s%ZENVIA_API_TOKEN%$ZENVIA_API_TOKEN%g" config.$ENVIRONMENT.yaml - sed -i "s/ENV/$ENVIRONMENT/g" config.$ENVIRONMENT.yaml - sed -i "s%RECAPTCHA_SECRET%$RECAPTCHA_SECRET%g" config.$ENVIRONMENT.yaml - sed -i "s%MONGODB_URI%$MONGODB_URI%g" config.$ENVIRONMENT.yaml - sed -i "s%ORY_SDK_URL%$ORY_SDK_URL%g" config.$ENVIRONMENT.yaml - sed -i "s%GITLAB_FEATURE_FLAG_URL%$GITLAB_FEATURE_FLAG_URL%g" config.$ENVIRONMENT.yaml - sed -i "s%GITLAB_FEATURE_FLAG_INSTANCE_ID%$GITLAB_FEATURE_FLAG_INSTANCE_ID%g" config.$ENVIRONMENT.yaml - sed -i "s/ORY_ACCESS_TOKEN/$ORY_ACCESS_TOKEN/g" config.$ENVIRONMENT.yaml - sed -i "s/ALETHEIA_SCHEMA_ID/$ALETHEIA_SCHEMA_ID/g" config.$ENVIRONMENT.yaml - sed -i "s%AWS_SDK_BUCKET%$AWS_SDK_BUCKET%g" config.$ENVIRONMENT.yaml - sed -i "s%AWS_ACCESS_KEY_ID%$AWS_ACCESS_KEY_ID%g" config.$ENVIRONMENT.yaml - sed -i "s%AWS_SECRET_ACCESS_KEY%$AWS_SECRET_ACCESS_KEY%g" config.$ENVIRONMENT.yaml - sed -i "s%NOVU_API_KEY%$NOVU_API_KEY%g" config.$ENVIRONMENT.yaml - sed -i "s%NOVU_APPLICATION_IDENTIFIER%$NOVU_APPLICATION_IDENTIFIER%g" config.$ENVIRONMENT.yaml - sed -i "s%AGENTS_API_URL%$AGENTS_API_URL%g" config.$ENVIRONMENT.yaml - sed -i "s%OPENAI_API_KEY%$OPENAI_API_KEY%g" config.$ENVIRONMENT.yaml - sed -i "s%AGENCIA_ACCESS_TOKEN%$AGENCIA_ACCESS_TOKEN%g" config.$ENVIRONMENT.yaml + pkl eval -f yaml ./deployment/config/config-file/$ENVIRONMENT.pkl > config.$ENVIRONMENT.yaml - name: Set migrate-mongo-config.ts run: | diff --git a/deployment/config/config-file/development.pkl b/deployment/config/config-file/development.pkl new file mode 100644 index 000000000..45fe80914 --- /dev/null +++ b/deployment/config/config-file/development.pkl @@ -0,0 +1,16 @@ +amends "./modules/main.pkl" +import "./modules/database/mongodb.pkl" +appBaseUrl = "aletheiafact.org" + +var = new { + name = "aletheia-development" + conf { + cors = "*" + recaptcha_sitekey = "6Lc2BtYUAAAAAOUBI-9r1sDJUIfG2nt6C43noOXh" + websocketUrl = "wss://testws.\(appBaseUrl) " + baseUrl = "https://test.\(appBaseUrl)" + db = (mongodb) { + atlas = true + } + } +} diff --git a/deployment/config/config-file/modules/agencia.pkl b/deployment/config/config-file/modules/agencia.pkl new file mode 100644 index 000000000..510c4fa1a --- /dev/null +++ b/deployment/config/config-file/modules/agencia.pkl @@ -0,0 +1 @@ +access_token = read("env:AGENCIA_ACCESS_TOKEN") diff --git a/deployment/config/config-file/modules/aws/s3.pkl b/deployment/config/config-file/modules/aws/s3.pkl new file mode 100644 index 000000000..b19e7599f --- /dev/null +++ b/deployment/config/config-file/modules/aws/s3.pkl @@ -0,0 +1,3 @@ +bucket = read("env:AWS_SDK_BUCKET") +accessKeyId = read("env:AWS_ACCESS_KEY_ID") +secretAccessKey = read("env:AWS_SECRET_ACCESS_KEY") diff --git a/deployment/config/config-file/modules/database/mongodb.pkl b/deployment/config/config-file/modules/database/mongodb.pkl new file mode 100644 index 000000000..121337739 --- /dev/null +++ b/deployment/config/config-file/modules/database/mongodb.pkl @@ -0,0 +1,8 @@ +atlas = false +connection_uri = read("env:MONGODB_URI") +options = new { + useUnifiedTopology = true + useNewUrlParser = true + retryWrites = true + w = "majority" +} diff --git a/deployment/config/config-file/modules/feature_flag.pkl b/deployment/config/config-file/modules/feature_flag.pkl new file mode 100644 index 000000000..710329557 --- /dev/null +++ b/deployment/config/config-file/modules/feature_flag.pkl @@ -0,0 +1,3 @@ +url = read("env:GITLAB_FEATURE_FLAG_URL") +appName = read("env:ENV") +instanceId = read("env:GITLAB_FEATURE_FLAG_INSTANCE_ID") diff --git a/deployment/config/config-file/modules/main.pkl b/deployment/config/config-file/modules/main.pkl new file mode 100644 index 000000000..f2e0961d1 --- /dev/null +++ b/deployment/config/config-file/modules/main.pkl @@ -0,0 +1,34 @@ +import "./database/mongodb.pkl" +import "./service/confBase.pkl" +import "./ory.pkl" as oryConfig + + +hidden appBaseUrl: String + +hidden var = new { + name = "service-aletheia" + conf = new { + port = 3000 + recaptcha_sitekey = read("env:RECAPTCHA_SITEKEY") + websocketUrl = read("env:WEBSOCKET_URL") + baseUrl = read("env:BASE_URL") + automatedFactCheckingAPIUrl = read("env:AGENTS_API_URL") + recaptcha_secret = read("env:RECAPTCHA_SECRET") + throttle = new confBase.Throttle { + ttl = 60 + limit = 100 + } + ory = (oryConfig) { + admin_endpoint = "admin" + } + } +} + +services = new Listing { + new { + name = var.name + conf = (confBase) { + ...var.conf + } + } +} diff --git a/deployment/config/config-file/modules/novu.pkl b/deployment/config/config-file/modules/novu.pkl new file mode 100644 index 000000000..171da3fac --- /dev/null +++ b/deployment/config/config-file/modules/novu.pkl @@ -0,0 +1,2 @@ +api_key = read("env:NOVU_API_KEY") +application_identifier = read("env:NOVU_APPLICATION_IDENTIFIER") diff --git a/deployment/config/config-file/modules/openai.pkl b/deployment/config/config-file/modules/openai.pkl new file mode 100644 index 000000000..3e4c05d6c --- /dev/null +++ b/deployment/config/config-file/modules/openai.pkl @@ -0,0 +1,2 @@ +api_key = read("env:OPENAI_API_KEY") + diff --git a/deployment/config/config-file/modules/ory.pkl b/deployment/config/config-file/modules/ory.pkl new file mode 100644 index 000000000..b804b978f --- /dev/null +++ b/deployment/config/config-file/modules/ory.pkl @@ -0,0 +1,5 @@ +url = read("env:ORY_SDK_URL") +admin_url = read("env:ORY_SDK_URL") +admin_endpoint: String +access_token = read("env:ORY_ACCESS_TOKEN") +schema_id = read("env:ALETHEIA_SCHEMA_ID") diff --git a/deployment/config/config-file/modules/service/base.pkl b/deployment/config/config-file/modules/service/base.pkl new file mode 100644 index 000000000..e30b8cc4f --- /dev/null +++ b/deployment/config/config-file/modules/service/base.pkl @@ -0,0 +1,4 @@ +class BaseService { + name: String + conf: Map +} diff --git a/deployment/config/config-file/modules/service/confBase.pkl b/deployment/config/config-file/modules/service/confBase.pkl new file mode 100644 index 000000000..7cd1811a4 --- /dev/null +++ b/deployment/config/config-file/modules/service/confBase.pkl @@ -0,0 +1,31 @@ +import "../database/mongodb.pkl" as mongoDbConfig +import "../ory.pkl" as oryConfig +import "../feature_flag.pkl" as featureFlagConfig +import "../aws/s3.pkl" as s3Config +import "../novu.pkl" as novuConfig +import "../openai.pkl" as openaiConfig +import "../zenvia.pkl" as zenviaConfig +import "../agencia.pkl" as agenciaConfig + +class Throttle { + ttl: Number + limit: Number +} + +port: Number +cors: String +websocketUrl: String +baseUrl: String +automatedFactCheckingAPIUrl: String +recaptcha_secret: String +recaptcha_sitekey: String +throttle: Throttle +authentication_type = "ory" +db = (mongoDbConfig) {} +ory = (oryConfig) {} +feature_flag = (featureFlagConfig) {} +aws = (s3Config) {} +novu = (novuConfig) {} +openai = (openaiConfig) {} +zenvia = (zenviaConfig) {} +agencia = (agenciaConfig) {} diff --git a/deployment/config/config-file/modules/zenvia.pkl b/deployment/config/config-file/modules/zenvia.pkl new file mode 100644 index 000000000..79c13ef73 --- /dev/null +++ b/deployment/config/config-file/modules/zenvia.pkl @@ -0,0 +1,2 @@ +api_url = read("env:ZENVIA_API_URL") +api_token = read("env:ZENVIA_API_TOKEN") diff --git a/deployment/config/config-file/production.pkl b/deployment/config/config-file/production.pkl new file mode 100644 index 000000000..33c525d20 --- /dev/null +++ b/deployment/config/config-file/production.pkl @@ -0,0 +1,16 @@ +amends "./modules/main.pkl" +import "./modules/database/mongodb.pkl" +appBaseUrl = "aletheiafact.org" + +var = new { + name = "aletheia-production" + conf { + cors = "*" + recaptcha_sitekey = "6Lc2BtYUAAAAAOUBI-9r1sDJUIfG2nt6C43noOXh" + websocketUrl = "wss://ws.\(appBaseUrl) " + baseUrl = "https://\(appBaseUrl)" + db = (mongodb) { + atlas = true + } + } +}