Merge main to crowd linux (#1786)

Co-authored-by: Mish Savelyev <[email protected]> Co-authored-by: Igor Kotua <[email protected]> Co-authored-by: Yeganathan S <[email protected]> Co-authored-by: Gasper Grom <[email protected]> Co-authored-by: anilb <[email protected]> Co-authored-by: Loïc Saint-Roch <[email protected]> Co-authored-by: Emmanuel Raymond <[email protected]> Co-authored-by: Sujal Ranjan <[email protected]> Co-authored-by: KRISH SONI <[email protected]> Co-authored-by: Akash Bag <[email protected]> Co-authored-by: Yash Yadav <[email protected]> Co-authored-by: Aditya Aryaman Das <[email protected]> Co-authored-by: Uros Marolt <[email protected]> Co-authored-by: Misha Savelyev <[email protected]>
CrowdDotDev · Oct 30, 2023 · 4cba93c · 4cba93c
1 parent 7eaf499
commit 4cba93c
Show file tree

Hide file tree

Showing 332 changed files with 209,284 additions and 30,926 deletions.
diff --git a/.github/workflows/lf-production-deploy-new.yaml b/.github/workflows/lf-production-deploy-new.yaml
@@ -15,6 +15,10 @@ on:
         description: Deploy webhook-api service?
         required: true
         type: boolean
+      deploy_automations_worker:
+        description: Deploy automations-worker service?
+        required: true
+        type: boolean
       deploy_script_executor:
         description: Deploy script-executor service?
         required: true
@@ -95,6 +99,27 @@ jobs:
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
 
+  build-and-push-automations-worker:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_automations_worker }}
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: automations-worker
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
   build-and-push-script-executor:
     runs-on: ubuntu-latest
     if: ${{ inputs.deploy_script_executor }}
@@ -170,6 +195,24 @@ jobs:
           image: ${{ needs.build-and-push-webhook-api.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
 
+  deploy-automations-worker:
+    needs: build-and-push-automations-worker
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_automations_worker }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: automations-worker
+          image: ${{ needs.build-and-push-automations-worker.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
   deploy-script-executor:
     needs: build-and-push-script-executor
     runs-on: ubuntu-latest

diff --git a/.github/workflows/lf-staging-deploy-automations-worker.yaml b/.github/workflows/lf-staging-deploy-automations-worker.yaml
@@ -0,0 +1,60 @@
+name: LF Staging Deploy Automations Worker
+
+on:
+  push:
+    branches:
+      - 'lf-staging/**'
+      - 'lf-staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/automations_worker/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.LF_STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.LF_STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.LF_AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.LF_AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.LF_AWS_REGION }}
+  SLACK_CHANNEL: deploys-lf-staging
+  SLACK_WEBHOOK: ${{ secrets.LF_STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: automations-worker
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-automations-worker:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: automations-worker
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
diff --git a/.github/workflows/production-deploy-new.yaml b/.github/workflows/production-deploy-new.yaml
@@ -15,6 +15,10 @@ on:
         description: Deploy webhook-api service?
         required: true
         type: boolean
+      deploy_automations_worker:
+        description: Deploy automations-worker service?
+        required: true
+        type: boolean
       deploy_script_executor:
         description: Deploy script-executor service?
         required: true
@@ -95,6 +99,27 @@ jobs:
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
 
+  build-and-push-automations-worker:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_automations_worker }}
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: automations-worker
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
   build-and-push-script-executor:
     runs-on: ubuntu-latest
     if: ${{ inputs.deploy_script_executor }}
@@ -170,6 +195,24 @@ jobs:
           image: ${{ needs.build-and-push-webhook-api.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
 
+  deploy-automations-worker:
+    needs: build-and-push-automations-worker
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_automations_worker }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: automations-worker
+          image: ${{ needs.build-and-push-automations-worker.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
   deploy-script-executor:
     needs: build-and-push-script-executor
     runs-on: ubuntu-latest

diff --git a/.github/workflows/staging-deploy-automations-worker.yaml b/.github/workflows/staging-deploy-automations-worker.yaml
@@ -0,0 +1,60 @@
+name: Staging Deploy Automations Worker
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+      - 'staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/automations-worker/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  SLACK_CHANNEL: deploys-staging
+  SLACK_WEBHOOK: ${{ secrets.STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: automations-worker
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-automations-worker:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: automations-worker
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -7,7 +7,7 @@ Contributions are what make the open source community such an amazing place to l
 - Try the crowd.dev platform & API and give feedback by [creating new issues](https://github.com/CrowdDotDev/crowd.dev/issues/new/choose)
 - Help with [open issues](https://github.com/CrowdDotDev/crowd.dev/issues)
 - Add a new integration following our [framework](https://docs.crowd.dev/docs/integration-framework)
-- Help create tutorials and blog posts
+- Help create tutorials and [blog](https://www.crowd.dev/blog) posts
 - Improve [documentation](https://docs.crowd.dev/docs) by fixing incomplete or missing docs, bad wording, examples or explanations
 
 Any contributions you make are **greatly appreciated**. ❤️
@@ -68,7 +68,7 @@ Any contributions you make are **greatly appreciated**. ❤️
 
 ## How to contribute to development
 
-We welcome any contribution to crowd.dev. Before you start with your first issue, please consider the folloing points:
+We welcome any contribution to crowd.dev. Before you start with your first issue, please consider the following points:
 
 - For your first contribution we recommend taking a look at our [good first issues 🥂](https://github.com/CrowdDotDev/crowd.dev/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue+%F0%9F%A5%82%22).
 - Other issues that are well suited for contribution have the tag [help wanted 🙏](https://github.com/CrowdDotDev/crowd.dev/labels/help%20wanted%20%F0%9F%99%8F).

diff --git a/README.md b/README.md
@@ -40,13 +40,13 @@
 - [Book a call](#📞-book-a-call)
 
 ## About crowd.dev
-crowd.dev is the developer data platform (DDP) that lets companies centralize all touch points developers have with their product and brand, be it in community (e.g. Stack Overflow or Reddit), product (open-source or SaaS), or commercial channels (e.g. HubSpot). The platform pulls data from a variety of different sources, normalizes it, matches identities across platforms, and enriches it with 3rd party data. The result is a unified 360-view of who the developers are that engage with your product and community, which companies they work for, and where they stand in their personal customer journey. 
+crowd.dev is the developer data platform (DDP) that lets companies centralize all touch points developers have with their product and brand, be it in the community (e.g. Stack Overflow or Reddit), product (open-source or SaaS), or commercial channels (e.g. HubSpot). The platform pulls data from a variety of different sources, normalizes it, matches identities across platforms, and enriches it with 3rd party data. The result is a unified 360-view of who the developers are that engage with your product and community, which companies they work for, and where they stand in their personal customer journey. 
 
 crowd.dev is open-source, built with developers in mind, available for both hosted and self-hosted deployments, open to extensions, and offers full control over your data. 
 
 **To our **users**:**
 - You can get actively involved, contribute to our roadmap, and turn crowd.dev into the tool you always wanted.
-- We are open regarding what we are building, allowing you to take a look inside, and making sure we handle your data in a privacy-preserving way.
+- We are open regarding what we are building, allowing you to take a look inside and making sure we handle your data in a privacy-preserving way.
 - You will never be locked in by us. Our interests as a company are aligned with you and we need to make sure that we always deliver enough value to you with our commercial offering in relation to our pricing.
 
 **To our developer community:**
@@ -63,7 +63,7 @@ crowd.dev is open-source, built with developers in mind, available for both host
 - **2-way CRM sync & Slack alerts** to get notified about intent events in real-time. [cloud only]
 - **User enrichment** with 25+ attributes, including emails, social profiles, work experience, and technical skills. [cloud only]
 - **Organization enrichment** with 50+ attributes, including industry, headcount, and revenue. [cloud only]
-- **Sentiment analysis and conversation detection** to stay on top of what's going on in your open source community. [cloud only]
+- **Sentiment analysis and conversation detection** to stay on top of what's going on in your open-source community. [cloud only]
 - **[Eagle Eye](https://www.crowd.dev/eagle-eye)**: Monitor dev-focused community platforms to find relevant content to engage with, helping you to gain developers’ mindshare and grow your community organically [cloud only]
 
 
@@ -83,14 +83,14 @@ Our services can be deployed using Kubernetes, as well as a lightweight developm
 
 #### Integrations
 
-We currently support all our integrations for self-hosting. For each one of them you will need to create your own application. You can see the steps for each integration in our [self-hosting integrations guide](https://docs.crowd.dev/docs/self-hosting).
+We currently support all our integrations for self-hosting. For each one of them, you will need to create your own application. You can see the steps for each integration in our [self-hosting integrations guide](https://docs.crowd.dev/docs/self-hosting).
 
 ### Development environment
 
 #### <a name="requirements">Requirements</a>
 
-- Node v16.16.0
-- Docker and docker-compose
+- [Node](https://nodejs.org/en) v16.16.0
+- [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/)
 
 #### <a name="getting_started">Getting started</a>
 
@@ -114,7 +114,7 @@ cd scripts
 ./cli clean-start-dev
 ```
 
-App will be available at http://localhost:8081
+The app will be available at http://localhost:8081
 
 For more information on development, you can <a href="https://docs.crowd.dev/docs/docker-compose-single-machine-development-with-docker-images">check our docs</a>.
 
@@ -134,7 +134,7 @@ There are many ways you can contribute to crowd.dev! Here are a few options:
 
 - Star this repo
 - Create issues every time you feel something is missing or goes wrong
-- Upvote issues with 👍 reaction so we know what's the demand for particular issue to prioritize it within roadmap
+- Upvote issues with 👍 reaction so we know what's the demand for a particular issue to prioritize it within the roadmap
 
 If you would like to contribute to the development of the project, please refer to our [Contributing guide](https://github.com/CrowdDotDev/crowd.dev/blob/main/CONTRIBUTING.md).
 

diff --git a/backend/.env.dist.composed b/backend/.env.dist.composed
@@ -22,4 +22,7 @@ CROWD_CUBEJS_URL="http://cubejs:4000/cubejs-api/v1"
 CROWD_NANGO_URL=http://nango:3003
 
 # OpenSearch settings
-CROWD_OPENSEARCH_NODE=http://open-search:9200
+CROWD_OPENSEARCH_NODE=http://open-search:9200
+
+# Temporal
+CROWD_TEMPORAL_SERVER_URL=temporal:7233
diff --git a/backend/.env.dist.local b/backend/.env.dist.local
@@ -166,3 +166,7 @@ CROWD_ANALYTICS_IS_ENABLED=
 CROWD_ANALYTICS_TENANT_ID=
 CROWD_ANALYTICS_BASE_URL=
 CROWD_ANALYTICS_API_TOKEN=
+
+# Temporal
+CROWD_TEMPORAL_SERVER_URL=localhost:7233
+CROWD_TEMPORAL_NAMESPACE=default
diff --git a/backend/.env.test b/backend/.env.test
@@ -5,4 +5,7 @@ CROWD_DB_PORT=5433
 CROWD_REDIS_PORT=6380
 
 # Sqs settings
-CROWD_SQS_PORT=9325
+CROWD_SQS_PORT=9325
+
+# Temporal settings
+CROWD_TEMPORAL_SERVER_URL=localhost:7234
diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json
@@ -199,5 +199,11 @@
     "tenantId": "CROWD_ANALYTICS_TENANT_ID",
     "baseUrl": "CROWD_ANALYTICS_BASE_URL",
     "apiToken": "CROWD_ANALYTICS_API_TOKEN"
+  },
+  "temporal": {
+    "serverUrl": "CROWD_TEMPORAL_SERVER_URL",
+    "namespace": "CROWD_TEMPORAL_NAMESPACE",
+    "certificate": "CROWD_TEMPORAL_CERTIFICATE",
+    "privateKey": "CROWD_TEMPORAL_PRIVATE_KEY"
   }
 }
diff --git a/backend/config/default.json b/backend/config/default.json
@@ -57,5 +57,8 @@
   "sso": {},
   "crowdAnalytics": {
     "isEnabled": "false"
+  },
+  "temporal": {
+    "automationsTaskQueue": "automations"
   }
 }
diff --git a/backend/docker-compose.test.yaml b/backend/docker-compose.test.yaml
@@ -42,6 +42,16 @@ services:
     networks:
       - crowd-bridge-test
 
+  temporal-test:
+    build:
+      context: ../scripts/scaffold/temporal
+    restart: always
+    ports:
+      - '7234:7233'
+      - '8234:8233'
+    networks:
+      - crowd-bridge-test
+
 networks:
   crowd-bridge-test:
     external: true