diff --git a/.github/workflows/deploy-alerting-temporal.yml b/.github/workflows/deploy-alerting-temporal.yml new file mode 100644 index 000000000000..86475310c2da --- /dev/null +++ b/.github/workflows/deploy-alerting-temporal.yml @@ -0,0 +1,52 @@ +name: Deploy Alerting Temporal + +on: + workflow_dispatch: + +concurrency: + group: deploy_alerting_temporal + cancel-in-progress: false + +env: + GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }} + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Get short sha + id: short_sha + run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: "Authenticate with Google Cloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCLOUD_SA_KEY }}" + + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v1" + + - name: Install gke-gcloud-auth-plugin + run: | + gcloud components install gke-gcloud-auth-plugin + + - name: Setup kubectl + run: | + gcloud container clusters get-credentials dust-kube --region us-central1 + + - name: Build the image on Cloud Build + run: | + chmod +x ./k8s/cloud-build.sh + ./k8s/cloud-build.sh alerting/temporal + + - name: Deploy the image on Kubernetes + run: | + chmod +x ./k8s/deploy-image.sh + ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/alerting-deployment-image:${{ steps.short_sha.outputs.short_sha }} alerting-temporal-deployment + + - name: Wait for rollout to complete + run: kubectl rollout status deployment/alerting-temporal-deployment --timeout=10m diff --git a/alerting/temporal/.gitignore b/alerting/temporal/.gitignore new file mode 100644 index 000000000000..3b0bb50e9f11 --- /dev/null +++ b/alerting/temporal/.gitignore @@ -0,0 +1,8 @@ +# dependencies +/node_modules + +# misc +.DS_Store +*.pem + + diff --git a/alerting/temporal/Dockerfile b/alerting/temporal/Dockerfile new file mode 100644 index 000000000000..7c6853c98f8b --- /dev/null +++ b/alerting/temporal/Dockerfile @@ -0,0 +1,7 @@ +# syntax=docker/dockerfile:1 + +FROM node:18-alpine +WORKDIR /src +COPY . . +RUN npm install +CMD ["npm", "run", "start"] \ No newline at end of file diff --git a/alerting/temporal/package-lock.json b/alerting/temporal/package-lock.json new file mode 100644 index 000000000000..b40f930fdb4a --- /dev/null +++ b/alerting/temporal/package-lock.json @@ -0,0 +1,437 @@ +{ + "name": "temporal-promql-to-dd.ts", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "temporal-promql-to-dd.ts", + "version": "0.1.0", + "dependencies": { + "@datadog/datadog-api-client": "^1.17.0", + "axios": "^1.5.1", + "zod": "^3.22.4" + }, + "devDependencies": { + "@tsconfig/node16": "^16.1.1", + "@types/node": "^20.8.2", + "ts-node": "^10.9.1" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@datadog/datadog-api-client": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-api-client/-/datadog-api-client-1.17.0.tgz", + "integrity": "sha512-iZuQmJDe+7PRzWZ40fYp/IfDJS9uNONBZvNNpjOjn1j9XplzlloJtyX1clRm/Hql7qHfK5+xrJR1HT4p1X/zRw==", + "dependencies": { + "@types/buffer-from": "^1.1.0", + "@types/node": "*", + "@types/pako": "^1.0.3", + "buffer-from": "^1.1.2", + "cross-fetch": "^3.1.5", + "es6-promise": "^4.2.8", + "form-data": "^4.0.0", + "loglevel": "^1.8.1", + "pako": "^2.0.4", + "url-parse": "^1.4.3" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", + "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "16.1.1", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-16.1.1.tgz", + "integrity": "sha512-+pio93ejHN4nINX4pXqfnR/fPLRtJBaT4ORaa5RH0Oc1zoYmo2B2koG+M328CQhHKn1Wj6FcOxCDFXAot9NhvA==", + "dev": true + }, + "node_modules/@types/buffer-from": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@types/buffer-from/-/buffer-from-1.1.0.tgz", + "integrity": "sha512-BLFpLBcN+RPKUsFxqRkMiwqTOOdi+TrKr5OpLJ9qCnUdSxS6S80+QRX/mIhfR66u0Ykc4QTkReaejOM2ILh+9Q==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "20.8.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.2.tgz", + "integrity": "sha512-Vvycsc9FQdwhxE3y3DzeIxuEJbWGDsnrxvMADzTDF/lcdR9/K+AQIeAghTQsHtotg/q0j3WEOYS/jQgSdWue3w==" + }, + "node_modules/@types/pako": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.5.tgz", + "integrity": "sha512-cg6x1RjMyCYoAdhOyNC/144EqhdHJXXZiiTgN3o+ZtOu4+ZQVN5msZgNyxzDI1w+dMYDdRamDzto3+bkR3FFQQ==" + }, + "node_modules/acorn": { + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", + "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, + "node_modules/cross-fetch": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", + "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", + "dependencies": { + "node-fetch": "^2.6.12" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/es6-promise": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" + }, + "node_modules/follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/loglevel": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz", + "integrity": "sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg==", + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/ts-node/node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, + "node_modules/typescript": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "dev": true, + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/zod": { + "version": "3.22.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", + "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/alerting/temporal/package.json b/alerting/temporal/package.json new file mode 100644 index 000000000000..e2a43dc1161a --- /dev/null +++ b/alerting/temporal/package.json @@ -0,0 +1,18 @@ +{ + "name": "temporal-promql-to-dd.ts", + "version": "0.1.0", + "private": true, + "scripts": { + "start": "ts-node src/index.ts" + }, + "devDependencies": { + "@tsconfig/node16": "^16.1.1", + "@types/node": "^20.8.2", + "ts-node": "^10.9.1" + }, + "dependencies": { + "@datadog/datadog-api-client": "^1.17.0", + "axios": "^1.5.1", + "zod": "^3.22.4" + } +} diff --git a/alerting/temporal/src/index.ts b/alerting/temporal/src/index.ts new file mode 100644 index 000000000000..116c0ba54e83 --- /dev/null +++ b/alerting/temporal/src/index.ts @@ -0,0 +1,320 @@ +import { client, v2 } from "@datadog/datadog-api-client"; +import axios from "axios"; +import { Agent } from "https"; +import { z } from "zod"; +import * as fs from "fs"; + +/** + * This service polls the Temporal Cloud Prometheus endpoint and submits the metrics to Datadog. + * + * Code is adapted from https://github.com/temporalio/samples-server/tree/main/cloud/observability/promql-to-dd-ts + */ +const TARGET_METRIC_NAMES = [ + "temporal_cloud_v0_workflow_terminate_count", + "temporal_cloud_v0_workflow_timeout_count", + "temporal_cloud_v0_workflow_failed_count", + "temporal_cloud_v0_workflow_cancel_count", + "temporal_cloud_v0_frontend_service_error_count", + "temporal_cloud_v0_resource_exhausted_error_count", + "temporal_cloud_v0_schedule_buffer_overruns_count", + "temporal_cloud_v0_schedule_rate_limited_count", +]; + +const requireEnvVar = (variableName: string): string => { + const value = process.env[variableName]; + if (!value) { + throw new Error(`Missing environment variable ${variableName}`); + } + return value; +}; + +// Required env variables +const METRICS_CLIENT_CERT = requireEnvVar("METRICS_CLIENT_CERT"); +const METRICS_CLIENT_KEY = requireEnvVar("METRICS_CLIENT_KEY"); +const TEMPORAL_CLOUD_BASE_URL = requireEnvVar("TEMPORAL_CLOUD_BASE_URL"); + +// This automatically pulls API keys from env vars DD_API_KEY +const configuration = client.createConfiguration(); +//End Required env variables + +const PROM_LABELS_URL = `${TEMPORAL_CLOUD_BASE_URL}/prometheus/api/v1/label/__name__/values`; +const PROM_QUERY_URL = `${TEMPORAL_CLOUD_BASE_URL}/prometheus/api/v1/query_range`; + +// This allows people to do local development on this service without polluting the metrics which +// are critical to ongoing LXB production observability. +const DATADOG_METRIC_PREFIX = process.env.TEST_METRIC_PREFIX || ""; + +// We're going to query Prometheus with a resolution of 1 minute +const PROMETHEUS_STEP_SECONDS = 60; + +// On an ongoing basis, query only for the last 1 minutes of data. +const QUERY_WINDOW_SECONDS = 1 * 60; + +const HISTOGRAM_QUANTILES = [0.5, 0.9, 0.95, 0.99]; + +configuration.setServerVariables({ + site: "datadoghq.eu", // We're using the EU site for Datadog +}); +const datadogMetricsApi = new v2.MetricsApi(configuration); +const logApi = new v2.LogsApi(configuration); + +const setTimeoutAsync = async (millis: number): Promise => { + return new Promise((resolve) => setTimeout(resolve, millis)); +}; + +const httpsAgent = new Agent({ + cert: fs.readFileSync(METRICS_CLIENT_CERT), + key: fs.readFileSync(METRICS_CLIENT_KEY), +}); + +const basePrometheusQueryParams = { + step: PROMETHEUS_STEP_SECONDS.toFixed(0), + format: "json", +}; + +const labelsResponseDataSchema = z.object({ + status: z.literal("success"), + data: z.string().array(), +}); + +const getMetricNames = async (): Promise<{ + countMetricNames: string[]; + histogramMetricNames: string[]; +}> => { + const metricNamesResponse = await axios.get(PROM_LABELS_URL, { + httpsAgent, + }); + + const { data: metricNames } = labelsResponseDataSchema.parse( + metricNamesResponse.data + ); + + const countMetricNames = metricNames.filter( + (metricName) => + TARGET_METRIC_NAMES.includes(metricName) && metricName.endsWith("_count") + ); + const histogramMetricNames = metricNames.filter( + (metricName) => + TARGET_METRIC_NAMES.includes(metricName) && metricName.endsWith("_bucket") + ); + + return { countMetricNames, histogramMetricNames }; +}; + +const queryResponseDataSchema = z.object({ + status: z.literal("success"), + data: z.object({ + resultType: z.literal("matrix"), + result: z + .object({ + metric: z.record(z.string()), + values: z.tuple([z.number(), z.string()]).array(), + }) + .array(), + }), +}); + +type MetricData = z.infer["data"]; + +type QueryWindow = { + startSecondsSinceEpoch: number; + endSecondsSinceEpoch: number; +}; + +const generateQueryWindow = (): QueryWindow => { + const endSecondsSinceEpoch = Date.now() / 1000; + + const windowInSeconds = QUERY_WINDOW_SECONDS; + const startSecondsSinceEpoch = endSecondsSinceEpoch - windowInSeconds; + return alignQueryWindowOnPrometheusStep({ + startSecondsSinceEpoch, + endSecondsSinceEpoch, + }); +}; + +// I'm not exactly sure why this is important, but I think that without it, we may inaccurately +// report some metrics. +const alignQueryWindowOnPrometheusStep = ( + queryWindow: QueryWindow +): QueryWindow => { + const startSecondsSinceEpoch = + Math.floor( + queryWindow.startSecondsSinceEpoch / PROMETHEUS_STEP_SECONDS - 1 + ) * PROMETHEUS_STEP_SECONDS; + const endSecondsSinceEpoch = + Math.floor(queryWindow.endSecondsSinceEpoch / PROMETHEUS_STEP_SECONDS + 1) * + PROMETHEUS_STEP_SECONDS; + + return { startSecondsSinceEpoch, endSecondsSinceEpoch }; +}; + +const queryPrometheusCount = async ( + metricName: string, + queryWindow: QueryWindow +): Promise => { + const response = await axios.get(PROM_QUERY_URL, { + httpsAgent, + params: { + ...basePrometheusQueryParams, + query: `rate(${metricName}[1m])`, + start: queryWindow.startSecondsSinceEpoch.toFixed(0), + end: queryWindow.endSecondsSinceEpoch.toFixed(0), + }, + }); + + return queryResponseDataSchema.parse(response.data).data; +}; + +const convertPrometheusCountToDatadogRateSeries = ( + metricName: string, + metricData: MetricData +): v2.MetricSeries[] => + metricData.result.map((prometheusMetric) => ({ + // We need to inform datadog of the interval for this metric + interval: PROMETHEUS_STEP_SECONDS, + // Make it easier for the datadog user to understand what this metric is + metric: DATADOG_METRIC_PREFIX + metricName.split("_count")[0] + "_rate1m", + // Type 2 is a "rate" metric + type: 2, + points: prometheusMetric.values.map(([timestamp, value]) => { + return { + timestamp: timestamp, + value: parseFloat(value), + }; + }), + tags: Object.entries(prometheusMetric.metric) + .filter(([key]) => key !== "__rollup__") + // Datadog tags can't be longer than 200 characters + .map(([key, value]) => `${key}:${value.substring(0, 200)}`), + })); + +const queryPrometheusHistogram = async ( + metricName: string, + quantile: number, + queryWindow: QueryWindow +): Promise => { + const response = await axios.get(PROM_QUERY_URL, { + httpsAgent, + params: { + ...basePrometheusQueryParams, + query: `histogram_quantile(${quantile}, sum(rate(${metricName}[1m])) by (temporal_account,temporal_namespace,operation,le))`, + start: queryWindow.startSecondsSinceEpoch.toFixed(0), + end: queryWindow.endSecondsSinceEpoch.toFixed(0), + }, + }); + + return queryResponseDataSchema.parse(response.data).data; +}; + +const convertPrometheusHistogramToDatadogGuageSeries = ( + metricName: string, + quantile: number, + metricData: MetricData +): v2.MetricSeries[] => + metricData.result.map((prometheusMetric) => ({ + // Make it easier for the datadog user to understand what this metric is + metric: + DATADOG_METRIC_PREFIX + + metricName.split("_bucket")[0] + + "_P" + + quantile * 100, + // Type 2 is a "guage" metric + type: 3, + points: prometheusMetric.values.map(([timestamp, value]) => { + return { + timestamp: timestamp, + value: parseFloat(value), + }; + }), + tags: Object.entries(prometheusMetric.metric) + .filter(([key]) => key !== "__rollup__") + // Datadog tags can't be longer than 200 characters + .map(([key, value]) => `${key}:${value.substring(0, 200)}`), + })); + +const main = async () => { + const { countMetricNames, histogramMetricNames } = await getMetricNames(); + + console.log({ + level: "info", + message: "Polling metrics", + countMetricNames, + histogramMetricNames, + }); + + while (true) { + const queryWindow = generateQueryWindow(); + + console.log({ + level: "info", + message: "Collecting metrics from temporal cloud.", + startDate: new Date( + queryWindow.startSecondsSinceEpoch * 1000 + ).toISOString(), + endDate: new Date(queryWindow.endSecondsSinceEpoch * 1000).toISOString(), + }); + + const countSeries = ( + await Promise.all( + countMetricNames.map(async (metricName) => + convertPrometheusCountToDatadogRateSeries( + metricName, + await queryPrometheusCount(metricName, generateQueryWindow()) + ) + ) + ) + ).flat(); + + const guageSeries = ( + await Promise.all( + histogramMetricNames.map(async (metricName) => + Promise.all( + HISTOGRAM_QUANTILES.map(async (quantile) => + convertPrometheusHistogramToDatadogGuageSeries( + metricName, + quantile, + await queryPrometheusHistogram( + metricName, + quantile, + generateQueryWindow() + ) + ) + ) + ) + ) + ) + ).flat(2); + + console.log({ level: "info", message: "Submitting metrics to Datadog" }); + await datadogMetricsApi.submitMetrics({ + body: { series: [...countSeries, ...guageSeries] }, + }); + + await logApi.submitLog({ + body: [ + { + ddsource: "temporal", + ddtags: "temporal", + hostname: "temporal", + message: "test message soupinou", + service: "temporal", + }, + ], + contentEncoding: "deflate", + }); + + const test = { series: [...countSeries, ...guageSeries] }; + console.log(JSON.stringify(test, null, 2)); + + console.log({ level: "info", message: "Pausing for 20s" }); + await setTimeoutAsync(20 * 1000); + } +}; + +main().catch((error) => { + console.log({ + level: "error", + message: "Error in main loop. Closing healthcheck server.", + error, + }); +}); diff --git a/k8s/apply_infra.sh b/k8s/apply_infra.sh index 35be868fd89d..8c793030eca8 100755 --- a/k8s/apply_infra.sh +++ b/k8s/apply_infra.sh @@ -97,6 +97,7 @@ apply_deployment connectors-edge-deployment apply_deployment blog-deployment apply_deployment docs-deployment apply_deployment metabase-deployment +apply_deployment alerting-temporal-deployment echo "-----------------------------------" diff --git a/k8s/configmaps/alerting-teporal.yaml b/k8s/configmaps/alerting-teporal.yaml new file mode 100644 index 000000000000..5be445ebfe4c --- /dev/null +++ b/k8s/configmaps/alerting-teporal.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: alerting-temporal-config +data: + DD_ENV: "prod" + DD_SERVICE: "alerting-temporal" + NODE_OPTIONS: "-r dd-trace/init" + DD_LOGS_INJECTION: "true" + DD_RUNTIME_METRICS_ENABLED: "true" + NODE_ENV: "production" diff --git a/k8s/deployments/alerting-temporal-deployment.yaml b/k8s/deployments/alerting-temporal-deployment.yaml new file mode 100644 index 000000000000..26837e8f452d --- /dev/null +++ b/k8s/deployments/alerting-temporal-deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alerting-temporal-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: alerting-temporal + template: + metadata: + labels: + app: alerting-temporal + name: alerting-temporal-pod + admission.datadoghq.com/enabled: "true" + annotations: + ad.datadoghq.com/web.logs: '[{"source": "alerting-temporal","service": "alerting-temporal","tags": ["env:prod"]}]' + spec: + containers: + - name: web + image: gcr.io/or1g1n-186209/alerting-temporal-image:latest + command: ["npm", "run", "start"] + imagePullPolicy: Always + envFrom: + - configMapRef: + name: alerting-temporal-config + - secretRef: + name: alerting-secrets + env: + - name: DD_AGENT_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + + volumeMounts: + - name: cert-volume + mountPath: /etc/certs + + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 500m + memory: 1Gi + volumes: + - name: cert-volume + secret: + secretName: temporal-datadog-cert