diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..23c75363 Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/api-service-build.yml b/.github/workflows/api-service-build.yml new file mode 100644 index 00000000..376d22e4 --- /dev/null +++ b/.github/workflows/api-service-build.yml @@ -0,0 +1,76 @@ +name: Build api service + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'src/backend/api/**' + - 'src/backend/core/**' + - 'src/backend/go.mod' + - '.github/workflows/api-service-build.yml' +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: API Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/backend + file: ./src/backend/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/apiservice:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + service=api + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update API server manifest + run: + sed -ie "s/apiservice:main/apiservice:${{env.GITHUB_SHA}}/g" ./src/backend/api/api-service-deployment.yaml + + - name: Deploy API server manifests + run: | + kubectl apply -f ./src/backend/api/api-service-deployment.yaml + + - name: Delete API server pod + run: | + kubectl rollout restart deploy/apiservice \ No newline at end of file diff --git a/.github/workflows/connector-service-build.yml b/.github/workflows/connector-service-build.yml new file mode 100644 index 00000000..294466ad --- /dev/null +++ b/.github/workflows/connector-service-build.yml @@ -0,0 +1,76 @@ +name: Build connector service + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'src/backend/connector/**' + - 'src/backend/core/**' + - 'src/backend/go.mod' + - '.github/workflows/connector-service-build.yml' +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Connector Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/backend + file: ./src/backend/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/connectorservice:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + service=connector + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update Connector server manifest + run: + sed -ie "s/connectorservice:main/connectorservice:${{env.GITHUB_SHA}}/g" ./src/backend/connector/service-deployment.yaml + + - name: Deploy Connector server manifests + run: | + kubectl apply -f ./src/backend/connector/service-deployment.yaml + + - name: Delete Connector server pod + run: | + kubectl rollout restart deploy/connectorservice \ No newline at end of file diff --git a/.github/workflows/embedder-build.yml b/.github/workflows/embedder-build.yml new file mode 100644 index 00000000..5e623361 --- /dev/null +++ b/.github/workflows/embedder-build.yml @@ -0,0 +1,78 @@ +name: Build embedder service + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'src/backend/embedder/**' + - '.github/workflows/embedder-build.yml' + +env: + # Use docker.io for Docker Hub if empty + #REGISTRY: ghcr.io + # github.repository as / + #IMAGE_NAME: ${{ github.repository }} + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Embedder Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/backend/embedder + file: ./src/backend/embedder/Dockerfile_cpu + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/embedder:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update Embedder server manifest + run: + sed -ie "s/embedder:main/embedder:${{env.GITHUB_SHA}}/g" ./src/backend/embedder/embedder-deployment.yaml + + - name: Deploy Embedder server manifests + run: | + kubectl apply -f ./src/backend/embedder/embedder-deployment.yaml + + - name: Delete Embedder server pod + run: | + kubectl rollout restart deploy/embedder diff --git a/.github/workflows/migration-service-build.yml b/.github/workflows/migration-service-build.yml new file mode 100644 index 00000000..aa74d2a3 --- /dev/null +++ b/.github/workflows/migration-service-build.yml @@ -0,0 +1,74 @@ +name: Build migration service + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'src/backend/migration/**' + - '.github/workflows/migration-service-build.yml' +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Migration Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/backend/migration + file: ./src/backend/migration/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/migrationservice:${{env.GITHUB_SHA}} + + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update migration server manifest + run: + sed -ie "s/migrationservice:main/migrationservice:${{env.GITHUB_SHA}}/g" ./src/backend/migration/migration-service-deployment.yaml + + - name: Deploy migration server manifests + run: | + kubectl apply -f ./src/backend/migration/migration-service-deployment.yaml + + - name: Delete migration server pod + run: | + kubectl rollout restart deploy/migrationservice diff --git a/.github/workflows/orchestrator-service-build.yml b/.github/workflows/orchestrator-service-build.yml new file mode 100644 index 00000000..c5a71ea6 --- /dev/null +++ b/.github/workflows/orchestrator-service-build.yml @@ -0,0 +1,71 @@ +name: Build orchestrator service + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'src/backend/orchestrator/**' + - 'src/backend/core/**' + - 'src/backend/go.mod' + - '.github/workflows/orchestrator-service-build.yml' +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Orchestrator Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/backend + file: ./src/backend/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/orchestratorservice:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + service=orchestrator + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update Orchestrator server manifest + run: + sed -ie "s/orchestratorservice:main/orchestratorservice:${{env.GITHUB_SHA}}/g" ./src/backend/orchestrator/service-deployment.yaml + + - name: Deploy Orchestrator server manifests + run: | + kubectl apply -f ./src/backend/orchestrator/service-deployment.yaml + + - name: Delete Orchestrator server pod + run: | + kubectl rollout restart deploy/orchestratorservice \ No newline at end of file diff --git a/.github/workflows/semantic-build.yml b/.github/workflows/semantic-build.yml new file mode 100644 index 00000000..b33050e1 --- /dev/null +++ b/.github/workflows/semantic-build.yml @@ -0,0 +1,73 @@ +name: Build semantic service + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'src/backend/semantic/**' + - '.github/workflows/semantic-build.yml' + +env: + # Use docker.io for Docker Hub if empty + #REGISTRY: ghcr.io + # github.repository as / + #IMAGE_NAME: ${{ github.repository }} + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Embedder Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/backend/semantic + file: ./src/backend/semantic/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/semantic:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update Semantic server manifest + run: + sed -ie "s/semantic:main/semantic:${{env.GITHUB_SHA}}/g" ./src/backend/semantic/semantic-deployment.yaml + + - name: Deploy Semantic server manifest + run: | + kubectl apply -f ./src/backend/semantic/semantic-deployment.yaml + + - name: Delete Semantic server pod + run: | + kubectl rollout restart deploy/semantic diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml new file mode 100644 index 00000000..c3e1595b --- /dev/null +++ b/.github/workflows/ui-build.yml @@ -0,0 +1,81 @@ +name: UI Build image + +on: + push: + branches: + - main + paths: + - 'src/web/embedder/**' + - '.github/workflows/ui-build.yml' + +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + +jobs: + build-ui-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - uses: benjlevesque/short-sha@v3.0 + id: short-sha + with: + length: 7 + + - run: echo $SHA + env: + SHA: ${{ steps.short-sha.outputs.sha }} + - run: echo $SHA + env: + SHA: ${{ env.SHA }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: UI Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./src/web + file: ./src/web/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/ui:${{env.SHA}} + build-args: | + COGNIX_VERSION=${{env.SHA}} + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - uses: azure/k8s-bake@v3 + with: + renderEngine: kustomize + kustomizationPath: ./src/web/deploy/ + kubectl-version: 'latest' + id: bake + + - uses: Azure/k8s-deploy@v5 + with: + namespace: 'default' + action: deploy + manifests: ${{ steps.bake.outputs.manifestsBundle }} + images: | + ${{ env.ACRNAME }}.azurecr.io/cognix/ui:${{env.SHA}} + imagepullsecrets: | + pullsecret diff --git a/.gitignore b/.gitignore index a547bf36..b7673cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,49 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-ssr -*.local - -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +.env + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +src/backend/vendor/ + +# Go workspace file +go.work + + + +# ignore Python cache files +__pycache__/ + +# ignore data path +data/ + +# pycharme + + +# ignore model path +src/backend/embedder/models/ + +# ignore temp path +src/backend/semantic/testdocs/ + +# ignore inteliJIdea configs +.idea/ + +# ignore temp path +vendor/ + + +# personal access token for one drive +docs/connectors/personal \ No newline at end of file diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 00000000..43e15c03 --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,60 @@ +
+ +# `CogniX` + + + +![Static Badge](https://img.shields.io/badge/mission-augment_humans_with_AI-orange) +
+![GitHub top language](https://img.shields.io/github/languages/top/gen-mind/cognix) +![GitHub last commit](https://img.shields.io/github/last-commit/gen-mind/cognix) + +[![Join our community on Slack](https://img.shields.io/badge/Join%20our%20community%20on%20Slack-Invite-brightgreen)](https://join.slack.com/t/cognixgroup/shared_invite/zt-2kbtcnys6-ZW3IjxeE~ZozZ8if6m0NWg) + + +[Join our community on Slack](https://join.slack.com/t/cognixgroup/shared_invite/zt-2kbtcnys6-ZW3IjxeE~ZozZ8if6m0NWg) + +## `What's CogniX` +Our mission is to augment humans with AI. + +## `What's CogniX` +AI can be difficult to use, CogniX fills the gap that there is between humans and AI. +And it augments it with all your knowledge on the palm of your hand. + +Imagine to be an engineer, facing an issue on the system. +Now it would be nice to quickly find my notes and related manuals to solve the issue.... +Probably you spent more time chasing your notes than actually solving the issue :) + +A teacher preparing lessons and exams for the week, or the semester + +Well with CogniX you can just type in: +- I need to solve + +being able to access information on your manuals, personal notes, + + +## `Setup` +Copy file config/.env.example to config/.env + +Declare env variables in config/.env + + +### `Run docker container` + +```azure + + docker compose up -d cockroach + docker compose up migration + docker compose up -d api +``` + + + +## `License` + +This project is dual-licensed: + +- **Non-Commercial Use**: Licensed under the Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0). See the [LICENSE](license) file for details. +- **Commercial Use**: Commercial licenses are available. Please see the [COMMERCIAL_LICENSE](license) file for details and contact [Your Contact Information] to discuss terms. + +For more information, visit [Cognix](https://www.cognix.ch). \ No newline at end of file diff --git a/deployment/.env.example b/deployment/.env.example new file mode 100644 index 00000000..061b7ee6 --- /dev/null +++ b/deployment/.env.example @@ -0,0 +1,36 @@ +# ***************************** +# Cognix # +# ***************************** +CONFIG_PATH=../src/config +DATA_PATH=../data +BACKEND_PATH=../src/backend +MIGRATION_PATH=../src/backend/migration +# ***************************** + +# ***************************** +# Milvus # +# ***************************** + +# to define environment variables available to docker-compose.yml +IMAGE_REPO=milvusdb +IMAGE_ARCH=amd64 +OS_NAME=ubuntu20.04 + +# for services.builder.image in docker-compose.yml +DATE_VERSION=20240429-6289f3a +LATEST_DATE_VERSION=20240429-6289f3a + +# added by gp to match milvus doker compose +DOCKER_VOLUME_DIRECTORY=/data +MILVUS_ROOT_PATH=/config/milvus.yaml + +# for services.gpubuilder.image in docker-compose.yml +GPU_DATE_VERSION=20240409-08bfb43 +LATEST_GPU_DATE_VERSION=20240409-08bfb43 + +# for other services in docker-compose.yml +MINIO_ADDRESS=minio:9000 +PULSAR_ADDRESS=pulsar://pulsar:6650 +ETCD_ENDPOINTS=etcd:2379 +AZURITE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" +# ***************************** \ No newline at end of file diff --git a/deployment/docker-compose-cognix-cpu.yaml b/deployment/docker-compose-cognix-cpu.yaml new file mode 100644 index 00000000..57107edf --- /dev/null +++ b/deployment/docker-compose-cognix-cpu.yaml @@ -0,0 +1,340 @@ +# cluster rag.cognix versions +# milvus:v2.4.0 +# nats:2.10.14 +# yugabytedb:2024.1.0.0-b129 + +# need to do give docker permissions on loval volumes +# id -gn # to find out the group name for your user: +# Use the group name obtained from the above command to set the correct permissions. Assuming the output of id -gn is staff, the command would be: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# Create the Required Directories: +# sudo mkdir -p /Users/gp/Developer/cognix/data/etcd +# sudo mkdir -p /Users/gp/Developer/cognix/data/minio +# sudo mkdir -p /Users/gp/Developer/cognix/data/yugabytedb +# sudo mkdir -p /Users/gp/Developer/cognix/data/nats +# sudo mkdir -p /Users/gp/Developer/cognix/data/milvus +# TODO: add the new folders needed for observability (all container involved) +# Set Correct Permissions: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chmod -R 755 YOUR_PATH/cognix/data/ +# Run Docker Compose with Sudo: +# sudo docker-compose -f deployment/docker-compose-test.yaml up + +# container versioning +# er use the semantic versioning as described here https://semver.org/ + +version: '3.8' +networks: + cognix-network: + # external: true + name: cognix-network + +services: + embedder: + container_name: embedder + image: gen-mind/cognix-embedder:latest + build: + context: ../src/backend/embedder + dockerfile: Dockerfile_cpu + ports: + - "50051:50051" + volumes: + - ${DATA_PATH}/models:/models + env_file: + - ${CONFIG_PATH}/embedder_srv.env + restart: always + networks: + - cognix-network + + semantic: + # container_name: semantic + image: gen-mind/cognix-semantic:latest + build: + context: ../src/backend/semantic + dockerfile: Dockerfile + volumes: + - ${DATA_PATH}/temp:/temp + env_file: + - ${CONFIG_PATH}/semantic_srv.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/yugabytedb_cli.env + - ${CONFIG_PATH}/minio_cli.env + restart: always +# ports: +# - "3434:8080" + healthcheck: + test: "curl --silent --fail http://localhost:8080/healthz > /dev/null || exit 1" + interval: 60s + start_period: 10s + timeout: 3s + retries: 3 + depends_on: + - milvus + - nats + - yugabytedb + - minio + networks: + - cognix-network + + cognix-web: + container_name: web-ui + image: gen-mind/cognix-web:latest + build: + context: ../src/web + dockerfile: Dockerfile.dev + ports: + - "5173:80" + environment: + VITE_PLATFORM_API_URL: http://api:8080 + env_file: + - ${CONFIG_PATH}/web_srv.env + depends_on: + - api + restart: always + networks: + - cognix-network + + api: + container_name: api + image: gen-mind/cognix-api:latest + build: + context: ${BACKEND_PATH} + args: + service: api + dockerfile: Dockerfile + ports: + - "8080:8080" + env_file: + - ${CONFIG_PATH}/api_srv.env + - ${CONFIG_PATH}/yugabytedb_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - yugabytedb + - minio + - milvus + - nats + restart: always + networks: + - cognix-network + + orchestrator: + container_name: orchestrator + image: gen-mind/cognix-orchestrator:latest + build: + context: ${BACKEND_PATH} + args: + service: orchestrator + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/orchestrator_srv.env + - ${CONFIG_PATH}/yugabytedb_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/oauth_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - yugabytedb + - minio + - nats + restart: always + networks: + - cognix-network + + connector: + container_name: connector + image: gen-mind/cognix-connector:latest + build: + context: ${BACKEND_PATH} + args: + service: connector + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/yugabytedb_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/oauth_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - yugabytedb + - minio + - nats + restart: always + networks: + - cognix-network + + migration: + container_name: migration + image: gen-mind/cognix-migration:latest + build: + context: ${MIGRATION_PATH} + dockerfile: Dockerfile + volumes: + - ${MIGRATION_PATH}/versions:/versions + env_file: + - ${CONFIG_PATH}/yugabytedb_cli.env + depends_on: + yugabytedb: + condition: service_healthy + networks: + - cognix-network + + minio: + container_name: minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - "../data/minio:/minio_data" + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + etcd: + container_name: etcd-1 + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - "../data/etcd:/etcd" + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + milvus: + container_name: milvus-standalone + image: milvusdb/milvus:v2.3.0 + command: ["milvus", "run", "standalone"] + security_opt: + - seccomp:unconfined + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + # - "../src/config/milvus_srv.yaml:/milvus/configs/milvus.yaml" # to pass config to milvus + - "../data/milvus:/var/lib/milvus" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + expose: + - 9091 + depends_on: + etcd: + condition: service_healthy + minio: + condition: service_healthy + networks: + - cognix-network + + milvus-attu: + container_name: milvus-attu + image: zilliz/attu:v2.4.0 + environment: + MILVUS_URL: milvus:19530 + ports: + - "13000:3000" + networks: + - cognix-network + + # Milvus Admin is unmantained, use Attu instead + # milvus-admin: + # image: milvusdb/milvus-insight:latest + # https://github.com/zilliztech/attu + + yugabytedb: + container_name: yugabytedb + image: yugabytedb/yugabyte:2024.1.0.0-b129 + ports: + - "5433:5433" + command: yugabyted start --enable_pg_parity_early_access --background=false + volumes: + - "../data/yugabytedb:/root/var/data" + healthcheck: + test: postgres/bin/pg_isready -h yugabytedb -p 5433 + interval: 20s + timeout: 5s + retries: 5 + networks: + - cognix-network + + # natsmq configuration. + # more detail: https://docs.nats.io/running-a-nats-service/configuration + nats: + image: nats:latest + container_name: nats + ports: + - "4222:4222" + - "8222:8222" + - "6222:6222" + expose: + - 8222 + volumes: + - nats-storage:/data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=/data/nats" +# healthcheck: +# test: echo $$(wget --server-response http://nats:8222/varz 2>&1 | grep '200 OK') | grep '200' || exit 1 +# interval: 20s +# timeout: 5s +# retries: 5 +# start_period: 40s + networks: + - cognix-network + + nats-dashboard: + image: mdawar/nats-dashboard + container_name: nats-dashboard + environment: + # The NATS monitoring server will be proxied on /proxy/* on the frontend. + # See: config/Caddyfile for the reverse proxy configuration. + REVERSE_PROXY_UPSTREAM: 'nats:8222' + volumes: + # Optional config file. + - ${CONFIG_PATH}/nats_config.json:/srv/config.json + ports: + - "8000:80" +# - target: 80 +# published: 8000 +# protocol: tcp + networks: + - cognix-network + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats diff --git a/deployment/docker-compose-cognix-gpu.yaml b/deployment/docker-compose-cognix-gpu.yaml new file mode 100644 index 00000000..e2916edc --- /dev/null +++ b/deployment/docker-compose-cognix-gpu.yaml @@ -0,0 +1,346 @@ +# cluster rag.cognix versions +# milvus:v2.4.0 +# nats:2.10.14 +# cockroach-operator:v2.12.0 which is using cockroach:v23.1.11 + +# need to do give docker permissions on loval volumes +# id -gn # to find out the group name for your user: +# Use the group name obtained from the above command to set the correct permissions. Assuming the output of id -gn is staff, the command would be: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# Create the Required Directories: +# sudo mkdir -p /Users/gp/Developer/cognix/data/etcd +# sudo mkdir -p /Users/gp/Developer/cognix/data/minio +# sudo mkdir -p /Users/gp/Developer/cognix/data/cockroach +# sudo mkdir -p /Users/gp/Developer/cognix/data/nats +# sudo mkdir -p /Users/gp/Developer/cognix/data/milvus +# TODO: add the new folders needed for observability (all container involved) +# Set Correct Permissions: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chmod -R 755 YOUR_PATH/cognix/data/ +# Run Docker Compose with Sudo: +# sudo docker-compose -f deployment/docker-compose-test.yaml up + +# container versioning +# er use the semantic versioning as described here https://semver.org/ + +version: '3.8' +networks: + cognix-network: + # external: true + name: cognix-network + +services: + embedder: + container_name: embedder + image: gen-mind/cognix-embedder:latest + build: + context: ../src/backend/embedder + dockerfile: Dockerfile_gpu +# deploy: +# resources: +# reservations: +# devices: +# - capabilities: [ gpu ] + ports: + - "50051:50051" + volumes: + - ${DATA_PATH}/models:/models + env_file: + - ${CONFIG_PATH}/embedder_srv.env + restart: always + networks: + - cognix-network + + semantic: + # container_name: semantic + image: gen-mind/cognix-semantic:latest + build: + context: ../src/backend/semantic + dockerfile: Dockerfile + volumes: + - ${DATA_PATH}/temp:/temp + env_file: + - ${CONFIG_PATH}/semantic_srv.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/minio_cli.env + restart: always +# ports: +# - "3434:8080" + healthcheck: + test: "curl --silent --fail http://localhost:8080/healthz > /dev/null || exit 1" + interval: 60s + start_period: 10s + timeout: 3s + retries: 3 + depends_on: + - milvus + - nats + - cockroach + - minio + networks: + - cognix-network + + cognix-web: + container_name: web-ui + image: gen-mind/cognix-web:latest + build: + context: ../src/web + dockerfile: Dockerfile.dev + ports: + - "5173:80" + environment: + VITE_PLATFORM_API_URL: http://api:8080 + env_file: + - ${CONFIG_PATH}/web_srv.env + depends_on: + - api + restart: always + networks: + - cognix-network + + api: + container_name: api + image: gen-mind/cognix-api:latest + build: + context: ${BACKEND_PATH} + args: + service: api + dockerfile: Dockerfile + ports: + - "8080:8080" + env_file: + - ${CONFIG_PATH}/api_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - cockroach + - minio + - milvus + - nats + restart: always + networks: + - cognix-network + + orchestrator: + container_name: orchestrator + image: gen-mind/cognix-orchestrator:latest + build: + context: ${BACKEND_PATH} + args: + service: orchestrator + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/orchestrator_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/oauth_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - cockroach + - minio + - nats + restart: always + networks: + - cognix-network + + connector: + container_name: connector + image: gen-mind/cognix-connector:latest + build: + context: ${BACKEND_PATH} + args: + service: connector + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/oauth_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - cockroach + - minio + - nats + restart: always + networks: + - cognix-network + + migration: + container_name: migration + image: gen-mind/cognix-migration:latest + build: + context: ${MIGRATION_PATH} + dockerfile: Dockerfile + volumes: + - ${MIGRATION_PATH}/versions:/versions + env_file: + - ${CONFIG_PATH}/cockroach_cli.env + depends_on: + - cockroach + networks: + - cognix-network + + minio: + container_name: minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - "../data/minio:/minio_data" + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + etcd: + container_name: etcd-1 + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - "../data/etcd:/etcd" + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + milvus: + container_name: milvus-standalone + image: milvusdb/milvus:v2.4.5 + command: ["milvus", "run", "standalone"] + security_opt: + - seccomp:unconfined + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + # - "../src/config/milvus_srv.yaml:/milvus/configs/milvus.yaml" # to pass config to milvus + - "../data/milvus:/var/lib/milvus" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + expose: + - 9091 + depends_on: + etcd: + condition: service_healthy + minio: + condition: service_healthy + networks: + - cognix-network + + milvus-attu: + container_name: milvus-attu + image: zilliz/attu:v2.4.0 + environment: + MILVUS_URL: milvus:19530 + ports: + - "13000:3000" + networks: + - cognix-network + + # Milvus Admin is unmantained, use Attu instead + # milvus-admin: + # image: milvusdb/milvus-insight:latest + # https://github.com/zilliztech/attu + + cockroach: + container_name: cockroach + image: cockroachdb/cockroach:v24.1.1 + ports: + - "26257:26257" + - "28080:8080" + # command: start-single-node --insecure + command: start-single-node --insecure --cache=1.5GiB --max-sql-memory=1.5GiB + volumes: + - "../data/cockroach:/cockroach/cockroach-data" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health?ready=1"] + interval: 20s + timeout: 5s + retries: 5 + networks: + - cognix-network + + # natsmq configuration. + # more detail: https://docs.nats.io/running-a-nats-service/configuration + nats: + image: nats:2.10.16-alpine + container_name: nats + ports: + - "4222:4222" + - "8222:8222" + - "6222:6222" + expose: + - 8222 + volumes: + - nats-storage:/data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=/data/nats" +# healthcheck: +# test: echo $$(wget --server-response http://nats:8222/varz 2>&1 | grep '200 OK') | grep '200' || exit 1 +# interval: 20s +# timeout: 5s +# retries: 5 +# start_period: 40s + networks: + - cognix-network + + nats-dashboard: + image: mdawar/nats-dashboard + container_name: nats-dashboard + environment: + # The NATS monitoring server will be proxied on /proxy/* on the frontend. + # See: config/Caddyfile for the reverse proxy configuration. + REVERSE_PROXY_UPSTREAM: 'nats:8222' + volumes: + # Optional config file. + - ${CONFIG_PATH}/nats_config.json:/srv/config.json + ports: + - "8000:80" +# - target: 80 +# published: 8000 +# protocol: tcp + networks: + - cognix-network + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats \ No newline at end of file diff --git a/deployment/docker-compose-cognix-observability-cpu.yaml b/deployment/docker-compose-cognix-observability-cpu.yaml new file mode 100644 index 00000000..5ae6ad3f --- /dev/null +++ b/deployment/docker-compose-cognix-observability-cpu.yaml @@ -0,0 +1,458 @@ +# cluster rag.cognix versions +# milvus:v2.4.0 +# nats:2.10.14 +# cockroach-operator:v2.12.0 which is using cockroach:v23.1.11 + +# need to do give docker permissions on loval volumes +# id -gn # to find out the group name for your user: +# Use the group name obtained from the above command to set the correct permissions. Assuming the output of id -gn is staff, the command would be: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# Create the Required Directories: +# sudo mkdir -p /Users/gp/Developer/cognix/data/etcd +# sudo mkdir -p /Users/gp/Developer/cognix/data/minio +# sudo mkdir -p /Users/gp/Developer/cognix/data/cockroach +# sudo mkdir -p /Users/gp/Developer/cognix/data/nats +# sudo mkdir -p /Users/gp/Developer/cognix/data/milvus +# TODO: add the new folders needed for observability (all container involved) +# Set Correct Permissions: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chmod -R 755 YOUR_PATH/cognix/data/ +# Run Docker Compose with Sudo: +# sudo docker-compose -f deployment/docker-compose-test.yaml up + +# container versioning +# er use the semantic versioning as described here https://semver.org/ + +version: '3.8' +networks: + cognix-network: + # external: true + name: cognix-network + +services: + embedder: + container_name: embedder + image: gen-mind/cognix-embedder:latest + build: + context: ../src/backend/embedder + dockerfile: Dockerfile_cpu + ports: + - "50051:50051" + volumes: + - ${DATA_PATH}/models:/models + env_file: + - ${CONFIG_PATH}/embedder_srv.env + restart: always + networks: + - cognix-network + + semantic: + # container_name: semantic + image: gen-mind/cognix-semantic:latest + build: + context: ../src/backend/semantic + dockerfile: Dockerfile + volumes: + - ${DATA_PATH}/temp:/temp + env_file: + - ${CONFIG_PATH}/semantic_srv.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/minio_cli.env + restart: always + ports: + - "3434:8080" + healthcheck: + test: "curl --silent --fail http://localhost:8080/healthz > /dev/null || exit 1" + interval: 60s + start_period: 10s + timeout: 3s + retries: 3 + depends_on: + - milvus + - nats + - cockroach + - minio + networks: + - cognix-network + + cognix-web: + container_name: web-ui + image: gen-mind/cognix-web:latest + build: + context: ../src/web + dockerfile: Dockerfile.dev + ports: + - "5173:80" + environment: + VITE_PLATFORM_API_URL: http://api:8080 + env_file: + - ${CONFIG_PATH}/web_srv.env + depends_on: + - api + restart: always + networks: + - cognix-network + + api: + container_name: api + image: gen-mind/cognix-api:latest + build: + context: ${BACKEND_PATH} + args: + service: api + dockerfile: Dockerfile + ports: + - "8080:8080" + env_file: + - ${CONFIG_PATH}/api_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - cockroach + - minio + - milvus + - nats + restart: always + networks: + - cognix-network + + orchestrator: + container_name: orchestrator + image: gen-mind/cognix-orchestrator:latest + build: + context: ${BACKEND_PATH} + args: + service: orchestrator + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/orchestrator_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - cockroach + - minio + - nats + restart: always + networks: + - cognix-network + + connector: + container_name: connector + image: gen-mind/cognix-connector:latest + build: + context: ${BACKEND_PATH} + args: + service: connector + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/connector_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/oauth_cli.env + - ${CONFIG_PATH}/.env + depends_on: + - cockroach + - minio + - nats + restart: always + networks: + - cognix-network + + migration: + container_name: migration + image: gen-mind/cognix-migration:latest + build: + context: ${MIGRATION_PATH} + dockerfile: Dockerfile + volumes: + - ${MIGRATION_PATH}/versions:/versions + env_file: + - ${CONFIG_PATH}/cockroach_cli.env + depends_on: + - cockroach + networks: + - cognix-network + + minio: + container_name: minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - "../data/minio:/minio_data" + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + etcd: + container_name: etcd-1 + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - "../data/etcd:/etcd" + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + milvus: + container_name: milvus-standalone + image: milvusdb/milvus:v2.3.0 + command: ["milvus", "run", "standalone"] + security_opt: + - seccomp:unconfined + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + # - "../src/config/milvus_srv.yaml:/milvus/configs/milvus.yaml" # to pass config to milvus + - "../data/milvus:/var/lib/milvus" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + expose: + - 9091 + depends_on: + etcd: + condition: service_healthy + minio: + condition: service_healthy + networks: + - cognix-network + + milvus-attu: + container_name: milvus-attu + image: zilliz/attu:v2.4.0 + environment: + MILVUS_URL: milvus:19530 + ports: + - "13000:3000" + networks: + - cognix-network + + # Milvus Admin is unmantained, use Attu instead + # milvus-admin: + # image: milvusdb/milvus-insight:latest + # https://github.com/zilliztech/attu + + cockroach: + container_name: cockroach + image: cockroachdb/cockroach:v24.1.1 + ports: + - "26257:26257" + - "28080:8080" + command: start-single-node --insecure + volumes: + - "../data/cockroach:/cockroach/cockroach-data" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health?ready=1"] + interval: 20s + timeout: 5s + retries: 5 + networks: + - cognix-network + + # natsmq configuration. + # more detail: https://docs.nats.io/running-a-nats-service/configuration + nats: + image: nats:latest + container_name: nats + ports: + - "4222:4222" + - "8222:8222" + - "6222:6222" + expose: + - 8222 + volumes: + - nats-storage:/data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=/data/nats" +# healthcheck: +# test: echo $$(wget --server-response http://nats:8222/varz 2>&1 | grep '200 OK') | grep '200' || exit 1 +# interval: 20s +# timeout: 5s +# retries: 5 +# start_period: 40s + networks: + - cognix-network + + nats-dashboard: + image: mdawar/nats-dashboard + container_name: nats-dashboard + environment: + # The NATS monitoring server will be proxied on /proxy/* on the frontend. + # See: config/Caddyfile for the reverse proxy configuration. + REVERSE_PROXY_UPSTREAM: 'nats:8222' + volumes: + # Optional config file. + - ${CONFIG_PATH}/nats_config.json:/srv/config.json + ports: + - "8000:80" +# - target: 80 +# published: 8000 +# protocol: tcp + networks: + - cognix-network + + nats-exporter: + image: natsio/prometheus-nats-exporter + # image: synadia/prometheus-nats-exporter + container_name: nats-exporter + hostname: nats-exporter + command: "-connz -varz -channelz -serverz -subz http://nats:8222" + ports: + - 7777:7777 + expose: + - 7777 + networks: + - cognix-network + + prometheus: + image: prom/prometheus + container_name: prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - 9090:9090 + restart: unless-stopped + volumes: + - ${DATA_PATH}/prometheus:/etc/prometheus + - prom_data:/prometheus + expose: + - 9090 + networks: + - cognix-network + + grafana: + image: grafana/grafana:latest #grafana/alpine + container_name: grafana + ports: + - 3001:3001 + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=grafana + - GF_SERVER_HTTP_PORT=3001 + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + volumes: + - ${DATA_PATH}/grafana:/etc/grafana/provisioning/datasources + networks: + - cognix-network + + loki: + container_name: loki + image: grafana/loki:2.9.2 + ports: + - "3100:3100" + expose: + - 3100 + command: -config.file=/etc/loki/loki-config.yaml + volumes: + - ${DATA_PATH}/loki:/etc/loki + networks: + - cognix-network + + # 7.6.24 to work use the following versions: + # - promtail latest + # - loky 2.9.2 + promtail: + container_name: promtail + image: grafana/promtail:latest + volumes: + - /var/log:/var/log + - /var/run/docker.sock:/var/run/docker.soc + - ${DATA_PATH}/promtail:/etc/promtail + command: -config.file=/etc/promtail/promtail-local-config.yaml + networks: + - cognix-network + + cadvisor: + image: gcr.io/cadvisor/cadvisor + container_name: cadvisor + restart: unless-stopped + privileged: true + ports: + - "8081:8080" + expose: + - 8080 + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + devices: + - /dev/kmsg + networks: + - cognix-network + + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + expose: + - 9100 + networks: + - cognix-network + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats + prom_data: + driver: local + driver_opts: + type: none + o: bind + device: ../data/prometheus \ No newline at end of file diff --git a/deployment/how-to-build.md b/deployment/how-to-build.md new file mode 100644 index 00000000..3c3e7bdf --- /dev/null +++ b/deployment/how-to-build.md @@ -0,0 +1,35 @@ +Create a .env file in the same directory as your docker-compose.yml: + +DATA_PATH=/users/gp/Developer/cognix-services/data +CONFIG_PATH=/absolute/path/to/config +BACKEND_PATH=/absolute/path/to/backend +MIGRATION_PATH=/absolute/path/to/migration + +sudo docker-compose -f deployment/docker-compose-test.yaml up/down + +sudo docker-compose -f deployment/docker-compose-test.yaml --build + +from the docker folder +docker build -f Dockerfile-embedder -t embedder:latest .. + +CONNECT INSIDE THE CONTAINER +docker exec -it /bin/bash +bin/pulsar-admin namespaces set-schema-compatibility-strategy --compatibility AUTO_CONSUME + +inside the container "df" command shows all volumes mounted + + + +Change to Project Root: +and then rund this command to +docker build -f docker/Dockerfile-embedder -t embedder:latest . +sudo docker build -f docker/Dockerfile-embedder -t embedder:latest --build-arg COGNIX_PATH=. . + +docker run embedder + +docker compose -f docker-compose-embedder.yml build + +docker compose -f docker-compose-embedder.yml up -d +docker compose -f docker/docker-compose-services-ai.yaml up -d + +docker compose -f docker-compose-embedder.yml logs -f embedder \ No newline at end of file diff --git a/deployment/old/docker-compose-cockroach.yaml b/deployment/old/docker-compose-cockroach.yaml new file mode 100644 index 00000000..4e327a27 --- /dev/null +++ b/deployment/old/docker-compose-cockroach.yaml @@ -0,0 +1,18 @@ +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + cognix-network: + driver: bridge + name: cognix-network + +services: + cockroach: + image: cockroachdb/cockroach:latest-v23.2 + ports: + - "26257:26257" + - "28080:8080" + command: start-single-node --insecure + volumes: + - "../data/cockroach:/cockroach/cockroach-data" + networks: + - cognix-network diff --git a/deployment/old/docker-compose-milvus-minio copy.yml b/deployment/old/docker-compose-milvus-minio copy.yml new file mode 100644 index 00000000..078f8f95 --- /dev/null +++ b/deployment/old/docker-compose-milvus-minio copy.yml @@ -0,0 +1,69 @@ +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + cognix-network: + external: + name: cognix-network + +services: + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.0 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + volumes: + - "../data/etcd:/etcd" + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + networks: + - cognix-network + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2020-12-03T00-03-10Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + volumes: + - "../data/minio:/minio_data" + command: minio server /minio_data + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + standalone: + container_name: milvus-standalone + image: milvusdb/milvus:v2.0.2 + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - "../data/milvus:/var/lib/milvus" + ports: + - "19530:19530" + depends_on: + - "etcd" + - "minio" + networks: + - cognix-network + + milvus-admin: + image: milvusdb/milvus-insight:latest + environment: + HOST_URL: http://localhost:8000 + MILVUS_URL: host.docker.internal:19530 + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "8000:3000" + networks: + - cognix-network diff --git a/deployment/old/docker-compose-milvus-minio.yml b/deployment/old/docker-compose-milvus-minio.yml new file mode 100644 index 00000000..fe6b0439 --- /dev/null +++ b/deployment/old/docker-compose-milvus-minio.yml @@ -0,0 +1,66 @@ +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + cognix-network: + external: + name: cognix-network + +services: + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.0 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + volumes: + - "../data/etcd:/etcd" + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + networks: + - cognix-network + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2020-12-03T00-03-10Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + volumes: + - "../data/minio:/minio_data" + command: minio server /minio_data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + standalone: + container_name: milvus-standalone + image: milvusdb/milvus:v2.0.2 + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - "../data/milvus:/var/lib/milvus" + ports: + - "19530:19530" + depends_on: + - "etcd" + - "minio" + networks: + - cognix-network + + milvus-admin: + image: milvusdb/milvus-insight:latest + environment: + HOST_URL: http://localhost:8000 + MILVUS_URL: host.docker.internal:19530 + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "8000:3000" + networks: + - cognix-network diff --git a/deployment/old/docker-compose-nats.yaml b/deployment/old/docker-compose-nats.yaml new file mode 100644 index 00000000..000f4d07 --- /dev/null +++ b/deployment/old/docker-compose-nats.yaml @@ -0,0 +1,39 @@ +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + cognix-network: + external: + name: cognix-network + +services: + nats: + image: nats:latest # Use the latest NATS image + container_name: nats + ports: + - "4222:4222" # Client connections to NATS + - "8222:8222" # HTTP monitoring port for NATS + - "6222:6222" # Cluster port for NATS + volumes: + - nats-storage:/data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=/data/nats" + networks: + - cognix-network + nats-streaming-ui: + build: . + image: piotrpersona/nats-ui:latest + ports: + - "8282:8282" + networks: + - cognix-network + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats diff --git a/deployment/old/docker-compose-services-ai-apple-silicon.yaml b/deployment/old/docker-compose-services-ai-apple-silicon.yaml new file mode 100644 index 00000000..3b6bdc53 --- /dev/null +++ b/deployment/old/docker-compose-services-ai-apple-silicon.yaml @@ -0,0 +1,161 @@ +version: '3.8' +networks: + cognix-network: + external: true + name: cognix-network + +services: + embedder: + + image: cognix/embedder:latest + build: + context: ../python/embedder/ + dockerfile: Dockerfile + container_name: embedder + ports: + - "50051:50051" + volumes: + - ../data/models:/models + env_file: + - ../config/embedder.env + restart: always + networks: + - cognix-network + + # semantic: + # build: + # context: ${COGNIX_PATH} + # dockerfile: docker/Dockerfile-semantic + # args: + # - COGNIX_PATH=${COGNIX_PATH} + # image: semantic:latest + # container_name: semantic + # volumes: + # - ${DATA_PATH}/models:/models + # env_file: + # - ${CONFIG_PATH}/semantic.env + # networks: + # - cognix-network + + + # ***************************** + # Milvus + # ***************************** + + # requirements RAM 16GB - 4+ vCPU + # https://milvus.io/docs/prerequisite-docker.md + + + etcd: + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-..}/volumes/etcd:/etcd + command: etcd -listen-peer-urls=http://127.0.0.1:2380 -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls=http://127.0.0.1:2380 --initial-cluster default=http://127.0.0.1:2380 + ports: + - "2379:2379" + - "2380:2380" + - "4001:4001" + + pulsar: + image: milvusdb/pulsar:v2.8.2-m1 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-..}/volumes/pulsar:/pulsar/data + environment: + # bin/apply-config-from-env.py script will modify the configuration file based on the environment variables + # nettyMaxFrameSizeBytes must be calculated from maxMessageSize + 10240 (padding) + - nettyMaxFrameSizeBytes=104867840 # this is 104857600 + 10240 (padding) + - defaultRetentionTimeInMinutes=10080 + - defaultRetentionSizeInMB=8192 + # maxMessageSize is missing from standalone.conf, must use PULSAR_PREFIX_ to get it configured + - PULSAR_PREFIX_maxMessageSize=104857600 + - PULSAR_GC=-XX:+UseG1GC + ports: + - "6650:6650" + - "18080:8080" + + minio: + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-..}/volumes/minio:/minio_data + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + azurite: + image: mcr.microsoft.com/azure-storage/azurite + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-..}/volumes/azurite:/data + command: azurite-blob --blobHost 0.0.0.0 + ports: + - "10000:10000" + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "6831:6831/udp" + - "4317:4317" # OLTP over gRPC + - "4318:4318" # OLTP over HTTP + - "16686:16686" # frontent + - "14268:14268" # jaeger.thirft + # ***************************** + + # ***************************** + # NATS + # ***************************** + nats: + image: nats:latest # Use the latest NATS image + container_name: nats + ports: + - "4222:4222" # Client connections to NATS + - "8222:8222" # HTTP monitoring port for NATS + - "6222:6222" # Cluster port for NATS + volumes: + - nats-storage:../data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=../data/nats" + networks: + - cognix-network + + # ***************************** + # cockroach + # ***************************** + cockroach: + image: cockroachdb/cockroach:latest-v23.2 + ports: + - "26257:26257" + - "28080:8080" + command: start-single-node --insecure + volumes: + - "../data/cockroach:/cockroach/cockroach-data" + networks: + - cognix-network + + # ***************************** + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats +# ***************************** + + diff --git a/deployment/old/docker-compose-services-ai.yaml b/deployment/old/docker-compose-services-ai.yaml new file mode 100644 index 00000000..6b2c6875 --- /dev/null +++ b/deployment/old/docker-compose-services-ai.yaml @@ -0,0 +1,323 @@ +# docker network create cognix-network +#if network does not exists +version: '3.8' +networks: + cognix-network: + # external: true + name: cognix-network + + +services: + embedder: + container_name: embedder + image: cognix/embedder:latest + build: + context: ../src/backend/embedder + dockerfile: Dockerfile + ports: + - "50051:50051" + volumes: + - ${DATA_PATH}/models:/models + env_file: + - ${CONFIG_PATH}/embedder.env + restart: always + networks: + - cognix-network + + semantic: + container_name: semantic + image: cognix/semantic:latest + build: + context: ../src/backend/semantic + dockerfile: Dockerfile + # ports: + # - "50051:50051" + # volumes: + # - ${DATA_PATH}/models:/models + env_file: + - ${CONFIG_PATH}/semantic.env + restart: always + networks: + - cognix-network + + # api: + # container_name: api + # image: cognix:api + # build: + # context: ${BACKEND_PATH} + # args: + # service: api + # dockerfile: Dockerfile + # environment: + # PORT: 8080 + # STORAGE_PATH: /storage_volume/data + # UI_URL: http://localhost:8080/ + # MILVUS_URL: host.docker.internal:19530 + # ports: + # - "8080:8080" + # env_file: + # - ${CONFIG_PATH}/cockroach.env + # - ${CONFIG_PATH}/minio.env + # - ${CONFIG_PATH}/nats.env + # - ${CONFIG_PATH}/embedder.env + # - ${CONFIG_PATH}/.env + # volumes: + # - ${BACKEND_PATH}:/backend + # extra_hosts: + # - "host.docker.internal:host-gateway" + # networks: + # - cognix-network + + # orchestrator: + # container_name: orchestrator + # image: cognix:orchestrator + # build: + # context: ${BACKEND_PATH} + # args: + # service: orchestrator + # dockerfile: Dockerfile + # env_file: + # - ${CONFIG_PATH}/cockroach.env + # - ${CONFIG_PATH}/nats.env + # - ${CONFIG_PATH}/pulsar.env + # - ${CONFIG_PATH}/.env + # extra_hosts: + # - "host.docker.internal:host-gateway" + # volumes: + # - ${BACKEND_PATH}:/backend + # networks: + # - cognix-network + + # connector: + # container_name: connector + # image: cognix:connector + # build: + # context: ${BACKEND_PATH} + # args: + # service: connector + # dockerfile: Dockerfile + # env_file: + # - ${CONFIG_PATH}/cockroach.env + # - ${CONFIG_PATH}/nats.env + # - ${CONFIG_PATH}/.env + # environment: + # MILVUS_URL: host.docker.internal:19530 + # extra_hosts: + # - "host.docker.internal:host-gateway" + # volumes: + # - ${BACKEND_PATH}:/backend + # networks: + # - cognix-network + + # migration: + # container_name: connector + # image: cognix:migration + # build: + # context: ${MIGRATION_PATH} + # dockerfile: Dockerfile + + # volumes: + # - ${MIGRATION_PATH}/versions:/versions + # env_file: + # - ${CONFIG_PATH}/cockroach.env + # networks: + # - cognix-network + + # ***************************** + # MinIO + # ***************************** + # minio: + # container_name: minio + # image: minio/minio:RELEASE.2020-12-03T00-03-10Z + # environment: + # MINIO_ACCESS_KEY: minioadmin + # MINIO_SECRET_KEY: minioadmin + # volumes: + # - "../data/minio:/minio_data" + # command: minio server /minio_data + # ports: + # - "9000:9000" + # healthcheck: + # test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + # interval: 30s + # timeout: 20s + # retries: 3 + # networks: + # - cognix-network + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + # - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data + - "../data/minio:/minio_data" + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + # ***************************** + + # ***************************** + # Milvus + # ***************************** + # etcd: + # container_name: etcd + # image: quay.io/coreos/etcd:v3.5.0 + # environment: + # - ETCD_AUTO_COMPACTION_MODE=revision + # - ETCD_AUTO_COMPACTION_RETENTION=1000 + # - ETCD_QUOTA_BACKEND_BYTES=4294967296 + # volumes: + # - "../data/etcd:/etcd" + # command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + # networks: + # - cognix-network + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - "../data/etcd:/etcd" + # - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cognix-network + + # standalone: + # container_name: milvus-standalone + # image: milvusdb/milvus:v2.3.0 + # command: ["milvus", "run", "standalone"] + # environment: + # ETCD_ENDPOINTS: etcd:2379 + # MINIO_ADDRESS: minio:9000 + # volumes: + # - "../data/milvus:/var/lib/milvus" + # ports: + # - "19530:19530" + # depends_on: + # - "etcd" + # - "minio" + # networks: + # - cognix-network + + standalone: + container_name: milvus-standalone + image: milvusdb/milvus:v2.3.0 + command: ["milvus", "run", "standalone"] + security_opt: + - seccomp:unconfined + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - "../data/milvus:/var/lib/milvus" + # - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus + # healthcheck: + # test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + # interval: 30s + # start_period: 90s + # timeout: 20s + # retries: 3 + ports: + - "19530:19530" + - "9091:9091" + depends_on: + - "etcd" + - "minio" + networks: + - cognix-network + + # milvus-admin: + # image: milvusdb/milvus-insight:latest + # environment: + # HOST_URL: http://localhost:8000 + # MILVUS_URL: host.docker.internal:19530 + # extra_hosts: + # - "host.docker.internal:host-gateway" + # ports: + # - "8000:3000" + # networks: + # - cognix-network + # ***************************** + + # ***************************** + # cockroach + # ***************************** + cockroach: + image: cockroachdb/cockroach:latest-v23.2 + ports: + - "26257:26257" + - "28080:8080" + command: start-single-node --insecure + volumes: + - "../data/cockroach:/cockroach/cockroach-data" + networks: + - cognix-network + # ***************************** + + # ***************************** + # NATS + # ***************************** + nats: + image: nats:latest # Use the latest NATS image + container_name: nats + ports: + - "4222:4222" # Client connections to NATS + - "8222:8222" # HTTP monitoring port for NATS + - "6222:6222" # Cluster port for NATS + volumes: + - nats-storage:/data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=/data/nats" + networks: + - cognix-network + # not working atm but let's see if it works in some time + # https://github.com/piotrpersona/nats-streaming-ui + # nats-streaming-ui: + # build: . + # image: piotrpersona/nats-ui:latest + # ports: + # - "8282:8282" + # networks: + # - cognix-network + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats +# ***************************** + + + + + +# milvus-standalone | [2024/05/26 23:27:26.246 +00:00] [WARN] [grpcclient/client.go:372] ["Server ID mismatch, start to reset connection"] [role=rootcoord] [error="rpc error: code = Unknown desc = expected=%!s(int64=410), actual=%!s(int64=411): server ID mismatch"] +# milvus-standalone | [2024/05/26 23:27:26.246 +00:00] [WARN] [client/client.go:133] ["ClientBase ReCall grpc first call get error "] [role=rootcoord] [address=] [error="err: rpc error: code = Unknown desc = expected=%!s(int64=410), actual=%!s(int64=411): server ID mismatch\n, /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace\n/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:378 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n/go/src/github.com/milvus-io/milvus/internal/distributed/rootcoord/client/client.go:133 github.com/milvus-io/milvus/internal/distributed/rootcoord/client.wrapGrpcCall[...]\n/go/src/github.com/milvus-io/milvus/internal/distributed/rootcoord/client/client.go:147 github.com/milvus-io/milvus/internal/distributed/rootcoord/client.(*Client).GetComponentStates\n/go/src/github.com/milvus-io/milvus/internal/util/componentutil/componentutil.go:34 github.com/milvus-io/milvus/internal/util/componentutil.WaitForComponentStates.func1\n/go/src/github.com/milvus-io/milvus/pkg/util/retry/retry.go:40 github.com/milvus-io/milvus/pkg/util/retry.Do\n/go/src/github.com/milvus-io/milvus/internal/util/componentutil/componentutil.go:58 github.com/milvus-io/milvus/internal/util/componentutil.WaitForComponentStates\n/go/src/github.com/milvus-io/milvus/internal/util/componentutil/componentutil.go:73 github.com/milvus-io/milvus/internal/util/componentutil.WaitForComponentHealthy\n/go/src/github.com/milvus-io/milvus/internal/distributed/datanode/service.go:285 github.com/milvus-io/milvus/internal/distributed/datanode.(*Server).init\n/go/src/github.com/milvus-io/milvus/internal/distributed/datanode/service.go:189 github.com/milvus-io/milvus/internal/distributed/datanode.(*Server).Run\n"] +# milvus-standalone | [2024/05/26 23:27:26.248 +00:00] [DEBUG] [sessionutil/session_util.go:559] ["SessionUtil GetSessions "] [prefix=rootcoord] [key=rootcoord] [address=172.18.0.8:53100] +# milvus-standalone | [2024/05/26 23:27:26.248 +00:00] [DEBUG] [client/client.go:109] ["RootCoordClient GetSessions success"] [address=172.18.0.8:53100] [serverID=410] +# milvus-standalone | [2024/05/26 23:27:26.248 +00:00] [WARN] [grpcclient/client.go:385] ["Server ID mismatch, start to reset connection"] [role=rootcoord] [error="rpc error: code = Unknown desc = expected=%!s(int64=410), actual=%!s(int64=411): server ID mismatch"] \ No newline at end of file diff --git a/deployment/old/docker-compose-services.yaml b/deployment/old/docker-compose-services.yaml new file mode 100644 index 00000000..796330ff --- /dev/null +++ b/deployment/old/docker-compose-services.yaml @@ -0,0 +1,83 @@ +version: '3.5' + +services: + api: + image: cognix:api + build: + context: ${BACKEND_PATH} + args: + service: api + dockerfile: Dockerfile + ports: + - "8080:8080" + env_file: + - ${CONFIG_PATH}/api_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/embedder_cli.env + - ${CONFIG_PATH}/.env + volumes: + - ${BACKEND_PATH}:/backend + networks: + - cognix-network + + orchestrator: + image: cognix:orchestrator + build: + context: ${BACKEND_PATH} + args: + service: orchestrator + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/orchestrator_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/.env + volumes: + - ${BACKEND_PATH}:/backend + networks: + - cognix-network + + connector: + image: cognix:connector + build: + context: ${BACKEND_PATH} + args: + service: connector + dockerfile: Dockerfile + env_file: + - ${CONFIG_PATH}/connector_srv.env + - ${CONFIG_PATH}/cockroach_cli.env + - ${CONFIG_PATH}/nats_cli.env + - ${CONFIG_PATH}/milvus_cli.env + - ${CONFIG_PATH}/minio_cli.env + - ${CONFIG_PATH}/oauth_cli.env + - ${CONFIG_PATH}/.env + volumes: + - ${BACKEND_PATH}:/backend + networks: + - cognix-network + + migration: + image: cognix:migration + build: + context: ${MIGRATION_PATH} + dockerfile: Dockerfile + + volumes: + - ${MIGRATION_PATH}/versions:/versions + env_file: + - ${CONFIG_PATH}/cockroach_cli.env + networks: + - cognix-network + +volumes: + db_volume: + nats-storage: + +networks: + cognix-network: + # external: true + name: cognix-network \ No newline at end of file diff --git a/deployment/old/docker-compose-web.yaml b/deployment/old/docker-compose-web.yaml new file mode 100644 index 00000000..37560126 --- /dev/null +++ b/deployment/old/docker-compose-web.yaml @@ -0,0 +1,52 @@ +# cluster rag.cognix versions +# milvus:v2.4.0 +# nats:2.10.14 +# cockroach-operator:v2.12.0 which is using cockroach:v23.1.11 + +# need to do give docker permissions on loval volumes +# id -gn # to find out the group name for your user: +# Use the group name obtained from the above command to set the correct permissions. Assuming the output of id -gn is staff, the command would be: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# Create the Required Directories: +# sudo mkdir -p /Users/gp/Developer/cognix/data/etcd +# sudo mkdir -p /Users/gp/Developer/cognix/data/minio +# sudo mkdir -p /Users/gp/Developer/cognix/data/cockroach +# sudo mkdir -p /Users/gp/Developer/cognix/data/nats +# sudo mkdir -p /Users/gp/Developer/cognix/data/milvus +# Set Correct Permissions: +# sudo chown -R $(whoami):staff /Users/gp/Developer/cognix/data +# Run Docker Compose with Sudo: +# sudo docker-compose -f deployment/docker-compose-test.yaml up + +version: '3.8' +networks: + cognix-network: + # external: true + name: cognix-network + +services: + + cognix-web: + container_name: web-ui + image: gen-mind/cognix-web:latest + build: + context: ../src/web + dockerfile: Dockerfile.cloud + ports: + - "5173:80" + environment: + VITE_PLATFORM_API_URL: http://api:8080 + env_file: + - ${CONFIG_PATH}/web_srv.env + networks: + - cognix-network + + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 00000000..cad2ac93 Binary files /dev/null and b/docs/.DS_Store differ diff --git a/docs/Database.md b/docs/Database.md new file mode 100644 index 00000000..d191444c --- /dev/null +++ b/docs/Database.md @@ -0,0 +1,15 @@ +in folder backend/ migration /version will created new file like a 20240606162757_new_script.sql +in this file + +`-- +goose Up +-- +goose StatementBegin + write queries for upgrade dataabse +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +write queries for downgrade database +-- +goose StatementEnd` + + +go orm https://github.com/go-pg/pg \ No newline at end of file diff --git a/docs/chat-flow.md b/docs/chat-flow.md new file mode 100644 index 00000000..4447768e --- /dev/null +++ b/docs/chat-flow.md @@ -0,0 +1,34 @@ +The user can interact with CogniX, start a new chat in two different ways (in the future will be expanded) + +1. Chat with your documents (requires_llm = true) +2. Search only (requires requires_llm=false) + + +The user asks a question (query): "How do I do get attention in Collaboard" + +Server receives the chat + +following are business logic methods: + +method name strem_response +argument query (string) +returns the stream + +strem_response calls vector_search + method name vector_search + argument query (string) + returns a list of documents + + It performs a query against Milvus, with userid and tennantid + The query result will contian all the matching documents inside Milvus. + +if equires_llm == true + It retrives the information regarding Persona, prompt and llm + + Top X documents retrieved from Milvus query will be sent to the LLM (LLM Endpoint + eventually api_key) embedded into the prompt + + LLM will stream the answer which will be streamed back to the client +if equires_llm == false + will send back only the list of documents + +Chat and response is saved into chat history \ No newline at end of file diff --git a/docs/connectors/connectors.md b/docs/connectors/connectors.md new file mode 100644 index 00000000..ffaedcb9 --- /dev/null +++ b/docs/connectors/connectors.md @@ -0,0 +1,109 @@ +### Supported MIME types +In list below mapping supported mime types to proto fileType +```json + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": proto.FileType_XLS, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": proto.FileType_DOC, + "application/pdf": proto.FileType_PDF, + "application/rtf": proto.FileType_RTF, + "text/rtf": proto.FileType_RTF, + "text/plain": proto.FileType_TXT, + "application/vnd.openxmlformats-officedocument.presentationml.presentation": proto.FileType_PPT, +``` + +### Parameter description +User should prepare json for each connector and copy it to field ConnectorSpecificConfig on page create/edit connector. + +#### Web + +```json +{ + "url": "", + "site_map": "", + "search_for_sitemap": false, + "url_recursive": false +} +``` + +#### File +```json +{ + "file_name": "", + "mime_type": "" +} +``` +mime type should be determined when user upload file to minio. + +Connector translate this parameter for a chunker service in next format +```json + "minio::" +``` + +#### OneDrive + +```json +{ + "folder": "", + "recursive": false, + "token": { + "access_token": "", + "expiry": "", + "refresh_token": "", + "token_type": "" + } +} +``` + +- folder : optional, folder name for scanning +- recursive : false - scan only given folder , true - scan nested folders +- token : OAuth token for access to ```one drive``` + +```NOTE``` +While UI is not ready + +You can get access using requests +```js +http://localhost:8080/api/oauth/microsoft/auth_url?redirect_url=http://localhost:8080 +``` +then go to the link from response +```json + +{ + "status": 200, + "data": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=>&scope=offline_access Files.Read.All Sites.ReadWrite.All&response_type=code&redirect_uri=http://localhost:8080/api/oauth/microsoft/callback" +} +``` +Sigin using microsoft account and grant permission + +copy token from response + +```json +{ + "status": 200, + "data": { + "id": "", + "email": "", + "name": "", + "given_name": "", + "family_name": "", + "access_token": "", + "refresh_token": "", + "token": { + "access_token": "", + "token_type": "", + "refresh_token": "", + "expiry": "" + } + } +} +``` +#### scope for oauth service + + - ChannelMessage.Read.All + - Chat.Read + - Chat.ReadBasic + - Team.ReadBasic.All + - TeamSettings.Read.All + - ChannelSettings.Read.All + - Channel.ReadBasic.All + - Group.Read.All + - Directory.Read.All diff --git a/docs/connectors/onedrive.md b/docs/connectors/onedrive.md new file mode 100644 index 00000000..3303a6ae --- /dev/null +++ b/docs/connectors/onedrive.md @@ -0,0 +1,102 @@ +# OneDrive + +## Setup +From CogniX UI navigate to connectors and create a new connector +Choose `OneDrive` +At step 2: +- Choose a name, it's just a description +- Fill the "Connector Specific Configration" with the json below filled with the corect data +- Refresh frequency in seconds is the delta of time that CogniX will use to start a new scan on your connected data source +- Connector credential, fill with a random number, it's not used + +```json +{ + "folder": "", + "recursive": false, + "token": { + "access_token": "", + "expiry": "", + "refresh_token": "", + "token_type": "" + } +} + + +``` + +json properties:
+**folder**
+string, optional if not set CogniX will analyze the whole drive
+example: older/chapter1 +
+**recursive**
+bool, (default false), you can omit. It indicates if CogniX shall analyze all the subfolder of the given path or not
+**token**
+The OAuth token you generate from OneDrive for CogniX to have access to the resource. Below a detailed description on how to get it + +Since the UI is still under construction you'll need to do some manual steps to get the OneDrive token. +This process will be automated with the UI evolution + +paste in your browser the following link if you are running CogniX on your private Docker deployment +```js + http://localhost:8080/api/oauth/microsoft/auth_url?redirect_url=http://localhost:8080 +``` + +If you are using CogniX from [rag.cognix.ch](https://rag.cognix.ch) +```js + https://rag.cognix.ch/api/oauth/microsoft/auth_url?redirect_url=http://rag.cognix.ch +``` + +once you paste the link above in the browser you will get a json. copy link
+you will get something similar to the json below:
+ +```json +https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=>&scope=offline_access Files.Read.All Sites.ReadWrite.All&response_type=code&redirect_uri=http://localhost:8080/api/oauth/microsoft/callback +``` + +paste the link as described above in a new browser window
. +Sign in using you microsoft account and grant permission to CogniX
+There's a checkbox you need to mark "Consent on behalf of your company"
+Click Accept
+ +You will be prompted with another json similar to the one described above
+Copy the token from the response
+The property named "access_token", "expiry": refresh_token": "", "token_type" and paste in the json provided above
+It might be a bit complex because access token and refresh token are very long string +Make sure to copy them properly +The token that you will receive will look like the sample below + +```json +{ + "status": 200, + "data": { + "id": "", + "email": "", + "name": "", + "given_name": "", + "family_name": "", + "access_token": "", + "refresh_token": "", + "token": { + "access_token": "", + "token_type": "", + "refresh_token": "", + "expiry": "" + } + } +} +``` + +Now you have a json filled with all the values CogniX needs.
+Paste it into the connector specific configuration
+ +**Refresh frequency** is in second it tells CogniX every each seconds it need to rescan the source. +Make it at least 86400 (one day in seconds)
+**connector credentials** +not used add a number + +## Microsoft Configuration +If you don't have a M365 account, you need to set it up. Instructions [here](https://learn.microsoft.com/en-us/microsoft-365/admin/simplified-signup/signup-business-standard?view=o365-worldwide#sign-up-for-microsoft-365-for-business) + +Once you have a M365 account +Go to our Azure subscription and create a new app registration \ No newline at end of file diff --git a/docs/connectors/personal/access_token_sample.json b/docs/connectors/personal/access_token_sample.json new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/docs/connectors/personal/access_token_sample.json @@ -0,0 +1,2 @@ + + diff --git a/docs/connectors/teams.md b/docs/connectors/teams.md new file mode 100644 index 00000000..18617492 --- /dev/null +++ b/docs/connectors/teams.md @@ -0,0 +1,108 @@ +#### Microsoft Teams Teams +This is the description for the Microsoft Teams Team and private and group chats connector + +## Setup +From CogniX UI navigate to connectors and create a new connector +Choose `Teams` +At step 2: +- Choose a name, it's just a description +- Fill the "Connector Specific Configration" with the json below filled with the corect data +- Refresh frequency in seconds is the delta of time that CogniX will use to start a new scan on your connected data source +- Connector credential, fill with a random number, it's not used + +```json +{ + "team": "", + "analyze_chats": true, + "channels": [], + "files": { + "folder": "", + "recursive": true + }, + "token": { + "access_token": "", + "expiry": "", + "refresh_token": "", + "token_type": "" + } +} +``` + +- team : name of team for analyzing. teams channel will not be analyzed if empty string. +- channels : name of channels for analyzing ( all channels will be analyzed if array is empty ) +- analyze_chats : true for analyze all private and group chats. +- token : OAuth token for access to ```ms teams``` +- files : + - folder : optional, folder name for scanning + - recursive : false - scan only given folder , true - scan nested folders + + +Since the UI is still under construction you'll need to do some manual steps to get the OneDrive token. +This process will be automated with the UI evolution + +Paste in your browser the following link if you are running CogniX on your private Docker deployment +```js + http://localhost:8080/api/oauth/microsoft/auth_url?redirect_url=http://localhost:8080 +``` + +If you are using CogniX from [rag.cognix.ch](https://rag.cognix.ch) +```js + https://rag.cognix.ch/api/oauth/microsoft/auth_url?redirect_url=https://rag.cognix.ch +``` + +once you paste the link above in the browser you will get a string. copy link
+you will get something similar to the json below:
+ +```json +https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=>&scope=offline_access Files.Read.All Sites.ReadWrite.All&response_type=code&redirect_uri=http://localhost:8080/api/oauth/microsoft/callback +``` + +paste the link as described above in a new browser window
. +Sign in using you microsoft account and grant permission to CogniX
+There's a checkbox you need to mark "Consent on behalf of your company"
+Click Accept
+ +You will be prompted with another json similar to the one described above
+This is a overly long string, though. +Be prepared with an editor so that you will be able to handle correctly the copy/paste operations +Copy the token from the response
+The property named "access_token", "expiry": refresh_token": "", "token_type" and paste in the json provided above
+It might be a bit complex because access token and refresh token are very long string +Make sure to copy them properly +The token that you will receive will look like the sample below + +```json +{ + "status": 200, + "data": { + "id": "", + "email": "", + "name": "", + "given_name": "", + "family_name": "", + "access_token": "", + "refresh_token": "", + "token": { + "access_token": "", + "token_type": "", + "refresh_token": "", + "expiry": "" + } + } +} +``` + +Now you have a json filled with all the values CogniX needs.
+Paste it into the connector specific configuration
+ +**Refresh frequency** is in second it tells CogniX every each seconds it need to rescan the source. +Make it at least 86400 (one day in seconds)
+**connector credentials** +not used add a number + +## Microsoft Configuration +If you don't have a M365 account, you need to set it up. Instructions [here](https://learn.microsoft.com/en-us/microsoft-365/admin/simplified-signup/signup-business-standard?view=o365-worldwide#sign-up-for-microsoft-365-for-business) + +Once you have a M365 account +Go to our Azure subscription and create a new app registration + diff --git a/docs/connectors/test.json b/docs/connectors/test.json new file mode 100644 index 00000000..b40cb6c3 --- /dev/null +++ b/docs/connectors/test.json @@ -0,0 +1,17 @@ + +{ + "channel": "Dev Team", + "topics": [], + "files": { + "folder": "", + "recursive": true + }, + "token": { + "access_token": "eyJ0eXAiOiJKV1QiLCJub25jZSI6IkpUY2lZZnJrRWpvVGI2WGdTcjUya21makdzOFFnQ0F5TWtCZUJnUUtVb0UiLCJhbGciOiJSUzI1NiIsIng1dCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyIsImtpZCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyJ9.eyJhdWQiOiIwMDAwMDAwMy0wMDAwLTAwMDAtYzAwMC0wMDAwMDAwMDAwMDAiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9hNjUyZmNlOS1mNGI3LTRjZDEtYThiYi02M2NkOGVlNDUxYWMvIiwiaWF0IjoxNzE4ODk1NTI1LCJuYmYiOjE3MTg4OTU1MjUsImV4cCI6MTcxODkwMDE5MiwiYWNjdCI6MCwiYWNyIjoiMSIsImFpbyI6IkFYUUFpLzhYQUFBQW43V2VQWklxWUFLdE5GOG9pODgrN1pHUEU0WmFyQVhWNWZkZThuMms1RjdGSkUzOXlkMGlZTEZXdEc5YXpjODBiZFRvVEtkT2F0dFIwUFE5ZW03eHRYMWtITmptV1dmQWFhMDFrdDh1bE1RcCtoSlpvSWdOZTMvWFRaMXhxY1Y0bkpyNWpKdHY3SldWSDZiWWtWM09tdz09IiwiYW1yIjpbInB3ZCIsIm1mYSJdLCJhcHBfZGlzcGxheW5hbWUiOiJDb2duaVggLSBSQUciLCJhcHBpZCI6ImVkN2RlODFlLTJhYTktNGY3ZS1hOThiLTg0ZjQ0NWM5OWMzYSIsImFwcGlkYWNyIjoiMSIsImZhbWlseV9uYW1lIjoiU2FudG9wYW9sbyIsImdpdmVuX25hbWUiOiJHaWFuIFBhb2xvIiwiaWR0eXAiOiJ1c2VyIiwiaXBhZGRyIjoiMmEwNDplZTQxOjM6MTE4NjpjOTNkOjU5OGM6NjA5NDo3M2ZlIiwibmFtZSI6IkdpYW4gUGFvbG8gU2FudG9wYW9sbyIsIm9pZCI6IjQ1YzhjMjU1LWM1NjEtNDU1NC04Y2U1LTVkOTlkZmQzMjJjNyIsIm9ucHJlbV9zaWQiOiJTLTEtNS0yMS0yMTIyMTg0NzE0LTM4MDczMDkyNDQtMzc4OTY3OTIwMy0xMTE3IiwicGxhdGYiOiI1IiwicHVpZCI6IjEwMDNCRkZEOThEN0MyNUMiLCJyaCI6IjAuQVM4QTZmeFNwcmYwMFV5b3UyUE5qdVJSckFNQUFBQUFBQUFBd0FBQUFBQUFBQUN3QUg0LiIsInNjcCI6IkNoYW5uZWwuUmVhZEJhc2ljLkFsbCBDaGFubmVsTWVzc2FnZS5SZWFkLkFsbCBDaGFubmVsU2V0dGluZ3MuUmVhZC5BbGwgQ2hhdC5SZWFkIENoYXQuUmVhZEJhc2ljIERpcmVjdG9yeS5SZWFkLkFsbCBGaWxlcy5SZWFkLkFsbCBHcm91cC5SZWFkLkFsbCBTZXJ2aWNlQWN0aXZpdHktT25lRHJpdmUuUmVhZC5BbGwgU2l0ZXMuUmVhZFdyaXRlLkFsbCBUZWFtLlJlYWRCYXNpYy5BbGwgVGVhbVNldHRpbmdzLlJlYWQuQWxsIFVzZXIuUmVhZCBwcm9maWxlIG9wZW5pZCBlbWFpbCIsInNpZ25pbl9zdGF0ZSI6WyJrbXNpIl0sInN1YiI6Ik93MDhCMnNhM1FHMFZaTTc5UWVWSFF6MUdnVkhDY2djMmFFMjY0UTBwZTgiLCJ0ZW5hbnRfcmVnaW9uX3Njb3BlIjoiRVUiLCJ0aWQiOiJhNjUyZmNlOS1mNGI3LTRjZDEtYThiYi02M2NkOGVlNDUxYWMiLCJ1bmlxdWVfbmFtZSI6IkdpYW5QYW9sby5TYW50b3Bhb2xvQGlidi5jaCIsInVwbiI6IkdpYW5QYW9sby5TYW50b3Bhb2xvQGlidi5jaCIsInV0aSI6Iml4d0hyVEYxbVVPX2MxWDYzMUJYQUEiLCJ2ZXIiOiIxLjAiLCJ3aWRzIjpbIjYyZTkwMzk0LTY5ZjUtNDIzNy05MTkwLTAxMjE3NzE0NWUxMCIsIjliODk1ZDkyLTJjZDMtNDRjNy05ZDAyLWE2YWMyZDVlYTVjMyIsImYyZWY5OTJjLTNhZmItNDZiOS1iN2NmLWExMjZlZTc0YzQ1MSIsImI3OWZiZjRkLTNlZjktNDY4OS04MTQzLTc2YjE5NGU4NTUwOSJdLCJ4bXNfaWRyZWwiOiI2IDEiLCJ4bXNfc3QiOnsic3ViIjoiMTlnUGhMV0RLYkYySEkzZUJIUnJ3UDE3UWY0SE5JOWRzTDZuRkU2SG5XayJ9LCJ4bXNfdGNkdCI6MTQ0NTk1NzE3OSwieG1zX3RkYnIiOiJFVSJ9.qHWDbB9eSQFVlHkuij7D5dXh6Ws2d0vGBaigQohCor5RzgpxA3d788FeLYpU4JaFV_s3OMXZMsdX44j-Qp9RL5Todkeo2dLDO6cVMbL_IGMeUFrsZrhc045QWbOZRzWe1AxKJjDMTW_a_U0_QPoD6B2B2FxUkAOBcJhoE5KThyQmGpLy8OsJYFkGxvzNNK-aA3X464-WqgnrOn1sLhIs8Re1cIx5YoFr8nxNCWs24pC1HyxQBcp7ZZfVEnPJtFERVHKuEu214H2s_1n_mnmVBTTQDtfesRSUlmHLnZbvBq2OewyBGR3_9ZMXRdOS4zxBCdCrBafFCEo-PrEu7XNQXQ", + "token_type": "Bearer", + "refresh_token": "0.AS8A6fxSprf00Uyou2PNjuRRrB7ofe2pKn5PqYuE9EXJnDqwAH4.AgABAwEAAAApTwJmzXqdR4BN2miheQMYAgDs_wUA9P9v3NyNyljV6cn5mi9Mo3z5trm1O4e-rmASyOKE64kzhq0BW1EoP1asPsasH03BtZsqKJbLAkDcoQ9_yfCjkINS2VpFK6b9DbQziP_d2ONbt8XHUbxadA8a5-F7ZcvDA5HL5FAHewhROKJ85x91fDrZiTafEGCIzyCcZhkY9khw5b1kOb8vwWyuB1gJ-kCtzo01y2RLjKRENSYJOs8PH6OqEZU1qy9zkSi213Fsq6R6yG18WoUH8c2vbZxTHDrRA0Kw-kTF_FZgFK4Vp_SCPWQnYsYZAhlvSFad4Qnx9tBrSmIpQPXF5hEVk9mDKsTAv0QiEdh2rS99k7LkOAfo7Sxyj12G3_-HQF4r1ETxj733OxaClkgE5NBdHBprcjZu9H9deQ5kR42Y1FcZr2SYdtBHyBTxdi_VWuPQf-BZmytxRQ_AEcoGx0Z_Yd0ozmepvCkk4QGqzvT4Cr2dFeKc36ijVEl_gSBdMAQ5NLdbkhMbH9amUrTGZjkt-SKO9MJSPYyK9d3d4jWiJVVqsPLq8xorYK99azbm3i3JLN6lCAbA_4OMckXxoj_Nel7Xvfcc9nB17k9zPjSgXn9krgC1J63RZuU0OF6DM5hungoSnCFazjOcLJRVBZb45S3MMenO9niFhG8DCs1IyriH-8aaWRnwTZIXb63MB9lqyLClPpBFfkMr3c02iWxTZr2kfzpUTpYjUMtpSJQbuP6Q_NCda6oM11vDNegTfxu_9y0rPQl3b8bZxb5vF-DOqW-y-S1q0e1O47oDRPMWyJw0bgEEjTxHFqheOW8rD6yuulZ3WYv_iSgN3rdV-tCih6aHc3E6sRDuwTuw2a4lksUzfWGA7SpCDX-LViTW4PfcFwvHJWQ5sWHvfO4C3p765vVy", + "expiry": "2024-06-20T16:16:31.953034988Z" + } +} + + diff --git a/docs/documents.md b/docs/documents.md new file mode 100644 index 00000000..b695859d --- /dev/null +++ b/docs/documents.md @@ -0,0 +1,53 @@ + +### Milvus schema +``` +| name | type | description | +| --- | --- | --- | +| id | int64 | primary key | +| document_id | int64 | id of document in cockroach database | +| content | json | text content expected format {"content":""}| +| vector | []float32 | vector array | + +``` +?? ***Need to define dimension fo vector array*** +for testing with open-ai i used 1536 + +??? Milvus schemas for user and tenant can be created +- when user signin first time. (golang) +- when first document will be processed in embedder service (python) + +```shell +cockroach table documents +( + id bigint default unique_rowid() not null primary key, + document_id varchar not null, + connector_id bigint not null references public.connectors, + link varchar, + signature text, + created_date timestamp default now() not null, + updated_date timestamp, + deleted_date timestamp, + status varchar(100) default 'new'::STRING not null +) + +supported document statuses + + StatusPending = "pending" + StatusChunking = "chunking" + StatusEmbedding = "embedding" + StatusComplete = "complete" + +``` + +### Document Statuses in connector flow + +#### golang connector service +- Set status ***pending*** for all documents stored in database when connector is started. +- Read document one by one. + - if document was not modified (check hash) set status ***complete*** and do not send message for chunking + - if document new or was changed set status ***chunking*** and send message for chunking +- WWhen all documents were read from source delete ( or mark for delete need to discuss ) all documents with status ***pending*** + +#### python services +- chunking service set status ***embedding*** and send message to embedding service +- embedding service set status ***complete*** and store content in milvus. diff --git a/docs/flows.md b/docs/flows.md new file mode 100644 index 00000000..d56ef97d --- /dev/null +++ b/docs/flows.md @@ -0,0 +1,289 @@ +The orchestrator should play an intelligent role in managing the flow of data between different components, rather than acting as a simple scheduler. Here are my thoughts on how to structure the orchestrator to meet these goals: + + +[Orchestrator](#orchestrator) + - [General Behavior](#general-behavior) + - [Multiple Orchestrator Instances](#multiple-orchestrator-instances) + - [Connector Status](#connector-status) + - [Statuses](#statuses) + - [Important Considerations](#important-considerations) +- [Rules for Re-scanning Connectors](#rules-for-re-scanning-connectors) + - [URL](#url) +- [File](#file) +- [OneDrive, Google Drive, and Other Cloud Drives](#onedrive-google-drive-and-other-cloud-drives) +- [MS Teams and Slack](#ms-teams-and-slack) +- [Refresh Frequency](#refresh-frequency) +[Connectors](#connectors) + - [General Behavior](#general-behavior-1) + - [URL and File](#url-and-file) + - [OneDrive, Google Drive, and Other Cloud Drives](#onedrive-google-drive-and-other-cloud-drives-1) +[Chunker](#chunker) +[Document Table Schema](#document-table-schema) + - [document](#document) + - [connector](#connector) + - [chunking_data](#chunking_data) + + + +The orchestrator should play an intelligent role in managing the flow of data between different components, rather than acting as a simple scheduler. Here are my thoughts on how to structure the orchestrator to meet these goals: + +1. **Intelligent Orchestrator**: The orchestrator should have the capability to decide whether a scan request should go directly to the chunker or through the connector, based on the type of source and its current status. + +2. **Direct Handling for Certain Sources**: For sources like files and URLs, it makes sense for the orchestrator to send requests directly to the chunker, bypassing the connector to reduce unnecessary traffic and workload. This can significantly optimize the system's efficiency, especially at scale. + +3. **Connector's Role for Complex Sources**: Connectors should handle more complex sources, such as OneDrive, Google Drive, MS Teams, and Slack, where there is a need to scan directories, verify credentials, or handle nested structures. The connector can then process these sources and decide what needs to be sent to the chunker. + +Here's a revised structure based on these principles: + +# Orchestrator +The orchestrator is responsible for monitoring the connector table in the relational database and intelligently deciding when it's time for a new analysis of the single connector. A single connector row represents a knowledge source the user decided to have analyzed by CogniX and stored in the Vector database. + +## General Behavior +- Monitor the connector table for changes. +- Decide when a new scan or analysis is needed. +- Send analysis requests intelligently to the Connector or Chunker, depending on the specific use case. +- Ensure no other processes are running before starting a new scan for that particular source. + +## Multiple Orchestrator Instances +TBD: Define how to handle multiple orchestrator instances. + +## Connector Status +The status of the connector is currently determined by a field in the connector table. In the future, the status will be retrieved from NATS. + +### Statuses: +- **Active**: Connector just created. +- **Pending Scan**: Orchestrator sent a message to Connector or Chunker to scan the source. +- **Working**: Set by Connector and Orchestrator, indicating that the Connector or Chunker is working on it. +- **Scan Completed Successfully**: Set by Chunker. +- **Scan Completed with Errors**: Set by Chunker. +- **Disabled**: No further scans will be started for this connector. +- **Unable to Process**: Set by Chunker, indicating a problem with the source. Only one email will be sent to the user reporting the issue. + +### Important Considerations: +- Determine actions for rows in the Pending or Working status for long time, which means the process has crashed without being able to update the status. +- Investigate if NATS can notify when a new item is added to the dead letter queue. If so, the orchestrator or another service should subscribe to this message and set the status to Unable to Process. + + +### Rules for Re-scanning Connectors + +- The connector is in one of the following statuses: + - "Active" + - "Scan Completed Successfully" + - "Scan Completed with Errors" +- The connector is **not** in the following statuses: + - "Pending Scan" + - "Working" + - "Disabled" + - "Unable to Process" +- The resulting date after adding the refresh frequency (refresh_freq) to the last update date is less than the current UTC time (now_utc()). +- One-Time Scan Connectors + - If the connector type is "YT" (YouTube) or "File", the connector should not be triggered again once it reaches an ending status (either "Scan Completed Successfully" or "Scan Completed with Errors"). +- Statuses that Prevent Scanning + - The orchestrator should never trigger a scan if the connector is in "Disabled" or "Unable to Process" status. + + +## URL +The user can connect a URL as a knowledge source, providing: +- URL (mandatory) +- Sitemap URL (optional) +- Option to scan all links on the page (optional) +- Option to search for a sitemap if not provided + +The Orchestrator will forward the request directly to Chunker for this file type. No file is stored in MinIO. +If the user deletes the source, the API will soft delete the row from the relational database and hard delete related entities from the vector database. + +## File +The user can upload a file as a knowledge source, providing: +- The file to be analyzed (mandatory) + +Files are uploaded to MinIO and scanned only once. Once the status is set to scan completed (with or without errors), no further analysis requests are issued. The Orchestrator will send a request directly to Chunker as soon as the file is uploaded correctly. + +## OneDrive, Google Drive, and Other Cloud Drives +The user can connect a cloud drive as a knowledge source, providing: +- Path to be analyzed (mandatory) +- Option to scan the path only or all subfolders +- Necessary credentials to access the path + +The Orchestrator will forward the request to Connector for this file type. + +## MS Teams and Slack +The user can upload a file as a knowledge source, providing: +- The file to be analyzed (mandatory) + +TBD: Define the full flow and how to handle the documents table. + +## Refresh Frequency +- **URL**: One week +- **OneDrive, Google Drive, and other cloud drives**: One week +- **MS Teams and Slack**: Daily, only new messages + + +## Sequence diagram +- Orchestrator monitors the connector table. +- Depending on the connector status, it sends a scan request to the connector or chunker. +- Connector processes the request for complex sources and interacts with chunker as needed. +- Chunker processes the data and updates the status. + +![plot](https://github.com/gen-mind/cognix/blob/feature/chunking-3/docs/media/orchestrator_sequence_diagram.png) + +## Componenet diagram +![plot](https://github.com/gen-mind/cognix/blob/feature/chunking-3/docs/media/orchestrator_componenet_diagram.png) + + + +# Connectors +The connector does not perform any actions on the vector database. It scans and downloads files from sources, storing them in MinIO, and determines if files need to be reloaded and re-chunked. + +## General Behavior +- Verify if files need to be reloaded and re-chunked. +- Send messages to Chunker only for modified or new data. + +## URL and File +The Connector does not handle these directly; requests are sent directly to Chunker by the Orchestrator. + +## OneDrive - Google Drive and other cloud drives +When the Connector receives a new message from the orchestrator it will +- Creates a GUID “chunking_session” that will be sent to each Chunking message sent by this operation. This way the Chunker will be able to understand when to set this process as completed +- Set the connector status from Pending scan to Working by Connector. +- It will scan the drive (given the rules from the orchestrator, all sub-folder or not) and get a list of path/file +- for each path/file item will check in the documents table if the item shall be sent to Chunker, depending on the hash comparison between database and file actually scanned. + - Update chunking_session with the new chunking_session + - if the item needs to be scanned (because is new or updated) sets the "analyzed" field to false + - if the item does not needs to be scanned sets the "analyzed" field to true + - (it is important to update the database for all the itmes before sending messages to NATS to avoid concurrency) +- delete (physically) all the documents in the database that are not anymore present in the original source +- Iterage again the list (after DB is updated for all the rows) + - if the item needs to be scanned (because is new or updated) send a message to chunker + +### Sequence diagram + +1. **Orchestrator to Connector:** The Orchestrator sends a new message to the Connector to start the process. +2. **Connector:** The Connector creates a GUID named "chunking_session" and sets the status to "Working." +3. **Connector to Drive:** The Connector scans the drive based on the given rules and retrieves a list of files and paths. +4. **Loop for each file/path:** + - **Connector to Database:** The Connector checks in the database if the item should be sent to the Chunker by comparing hashes. + - **Database to Connector:** The database returns the hash comparison result. + - **Alt (Condition):** + - If the item needs to be scanned, the Connector updates the chunking_session and sets the "analyzed" field to false. + - If the item does not need to be scanned, the Connector sets the "analyzed" field to true. +5. **Connector to Database:** The Connector deletes documents from the database that are no longer present in the original source. +6. **Loop for each file/path:** + - **Alt (Condition):** + - If the item needs to be scanned, the Connector sends a message to the Chunker. +![plot](https://github.com/gen-mind/cognix/blob/feature/chunking-3/docs/media/connecor_cloud_drive_sequence_diagram.png) + + +# Chunker +Chunker processes data from various sources, creating embeddings and storing them in the vector database. + +## URL Chunker +Becaus ein golang is pretty time consuming creating a crawler able to analyze the content of a +This specific chunker has two +guid “chunking_session” + + +## OneDrive, Google Drive, and Other Cloud Drives + +# Document Table Schema +All the tables in the database shall have the following fields: +```sql + "creation_date" timestamp NOT NULL DEFAULT (now()), + "last_update" timestamp +``` +TBD if to add also ubdated_by (the user or the service that performed the last update) + + +## document +```sql +CREATE TABLE "document" ( + "id" varchar PRIMARY KEY NOT NULL, + "parent_id" integer, -- Allows nulls, used for URLs + "connector_id" integer NOT NULL, + "link" varchar, + "signature" text, + "chunking_session" guid, -- Allows nulls + "analyzed" bit -- default false, true when chunker created the embeddings in the vector db + "creation_date" timestamp NOT NULL DEFAULT (now()), --datetime utc IMPORTANT now() will not get the utc date!!!! + "last_update" timestamp --datetime utc +); +``` +The parent_id is used for URL. +The orchestrator does not know how may related URL will be found in the starting URL, to store all the data we need to use the parent id +The first document, is the URL the user provided. +URL Chunker will then create as many documents, with the parent_id set to the id of the document the user stored in the connector + +## connector +```sql +CREATE TABLE "connectors" ( + "id" SERIAL PRIMARY KEY, + "credential_id" integer NOT NULL, -- remove this and related tabe + "name" varchar NOT NULL, + "type" varchar(50) NOT NULL, -- PDF, URL etc + "status" varchar, + "connector_specific_config" jsonb NOT NULL, + "refresh_freq" integer, + "user_id" uuid NOT NULL, + "tenant_id" uuid NOT NULL, + "last_successful_analyzed" timestamp, --datetime utc + "total_docs_analyzed" integer, + "creation_date" timestamp NOT NULL DEFAULT (now()), --datetime utc + "last_update" timestamp --datetime utc +); +``` +status in the connector will have the following values: +```sql + ConnectorStatusReadyToProcessed = "READY_TO_BE_PROCESSED" + ConnectorStatusPending = "PENDING" + ConnectorStatusWorking = "PROCESSING" + ConnectorStatusSuccess = "COMPLETED_SUCCESSFULLY" + ConnectorStatusError = "COMPLETED_WITH_ERRORS" + ConnectorStatusDisabled = "DISABLED" + ConnectorStatusUnableProcess = "UNABLE_TO_PROCESS" +``` + +Removed the shared field. If a connector has the tennat_id it means it is shared, if it has the user_id it means is from the user + +## chunking_data +```proto +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; + + +enum FileType { + UNKNOWN = 0; + URL = 1; + PDF = 2; + RTF = 3; + DOC = 4; + XLS = 5; + PPT = 6; + TXT = 7; + MD = 8; + YT = 9; + // add all supported file that in another document + // check what with Google docs +}; + +message ChunkingData { + // This is the url where the file is located. + // Based on the chunking type it will be a WEB URL (HTML type) + // Will be an S3/MINIO link with a proper authentication in case of a file + string url = 1; + string site_map = 2; + bool search_for_sitemap = 3; + int64 document_id = 4; + FileType file_type = 5; + string collection_name = 6; + string model_name = 7; + int32 model_dimension = 8; + uuid chunking_session = 9; + bool is_internal = 10; // used by chunker if it needs to send message to itself +} +``` + +- URL (mandatory) +- Sitemap URL (optional) +- Option to scan all links on the page (optional) +- Option to search for a sitemap if not provided diff --git a/docs/kind-milvus.md b/docs/kind-milvus.md new file mode 100644 index 00000000..4c041376 --- /dev/null +++ b/docs/kind-milvus.md @@ -0,0 +1,61 @@ + +#### Start kind cluster +```shell +cat <" + +Files from the public connector will be stored in this bucket directly, +while files from the private connector will be stored in a folder named "user-< UUID >" within the tenant's bucket. \ No newline at end of file diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 00000000..46b2af1b --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,4 @@ +https://www.giantswarm.io/blog/grafana-logging-using-loki + + +https://grafana.com/blog/2018/12/12/loki-prometheus-inspired-open-source-logging-for-cloud-natives/ \ No newline at end of file diff --git a/docs/pp b/docs/pp new file mode 100644 index 00000000..79a43838 --- /dev/null +++ b/docs/pp @@ -0,0 +1,3 @@ +{"content":"Collaboard Help CenterEnglishDeutschGo to collaboard.appSign inDeutsch + + EnglishDeutsch"} \ No newline at end of file diff --git a/docs/standards.md b/docs/standards.md new file mode 100644 index 00000000..626f459c --- /dev/null +++ b/docs/standards.md @@ -0,0 +1,98 @@ +# Standards + +**Containers + +| image names | deployment names | service names | pod names | +| ------------------- | ------------------- | ------------------- | ------------------- | +| cognix-embedder | cognix-embedder | cognix-embedder | cognix-embedder | +| cognix-semantic | cognix-semantic | cognix-semantic | cognix-semantic | +| cognix-web | cognix-web | cognix-web | cognix-web | +| cognix-api | cognix-api | cognix-api | cognix-api | +| cognix-orchestrator | cognix-orchestrator | cognix-orchestrator | cognix-orchestrator | +| cognix-connector | cognix-connector | cognix-connector | cognix-connector | +| cognix-migration | cognix-migration | cognix-migration | cognix-migration | + +**Configmaps (src/config) + +must end with -cli for client related parameters +must end with -srv for server related parameters +env-configmap for the global parameters (.env in docker) + +| name | +| ---------------- | +| api-srv | +| cockroach-cli | +| connector-srv | +| embedder-cli | +| embedder-srv | +| env-configmap | +| milvus-cli | +| minio-cli | +| nats-cli | +| oauth-cli | +| orchestrator-srv | +| semantic-srv | +| web-srv | + +**Github Actions workflows + + +| workflow file name | **workflow name** | +| ----------------------------- | ------------------------- | +| build-cognix-embedder.yml | Build Cognix Embedder | +| build-cognix-semantic.yml | Build Cognix Semantic | +| build-cognix-web.yml | Build Cognix Web | +| build-cognix-api.yml | Build Cognix API | +| build-cognix-orchestrator.yml | Build Cognix Orchestrator | +| build-cognix-connector.yml | Build Cognix Connector | +| build-cognix-migration.yml | Build Cognix Migration | + + +**Versioning, based on semantic versioning + +| image names | tag | +| ------------------- | ------ | +| cognix-embedder | v0.0.1 | +| cognix-semantic | v0.0.1 | +| cognix-web | v0.0.1 | +| cognix-api | v0.0.1 | +| cognix-orchestrator | v0.0.1 | +| cognix-connector | v0.0.1 | +| cognix-migration | v0.0.1 | + +Volumes + +| service names | volume name | pvc name | directory | shared | size | +| ------------------- | --------------------- | ---------------------- | --------- | ------ | ---- | +| cognix-embedder | models-volume | embedder-models-volume | /models | yes | ?? | +| cognix-semantic | use ephemeral storage | N/A | /temp | no | N/A | +| cognix-web | N/A | N/A | N/A | N/A | N/A | +| cognix-api | N/A | N/A | N/A | N/A | N/A | +| cognix-orchestrator | N/A | N/A | N/A | N/A | N/A | +| cognix-connector | N/A | N/A | N/A | N/A | N/A | +| cognix-migration | migration-volume | migration-volume | /versions | no | 20GB | + +Folders + +| service names | folder | +| ------------------- | ------------------------ | +| cognix-embedder | src/backend/embedder | +| cognix-semantic | src/backend/semantic | +| cognix-web | src/web | +| cognix-api | src/backend/api | +| cognix-orchestrator | src/backend/orchestrator | +| cognix-connector | src/backend/connector | +| cognix-migration | src/backend/migration | + + +Ports + +| service names | Port | +| ------------------- | ----- | +| cognix-embedder | 50051 | +| cognix-semantic | 8080 | +| cognix-web | 8080 | +| cognix-api | 8080 | +| cognix-orchestrator | none | +| cognix-connector | none | +| cognix-migration | none | \ No newline at end of file diff --git a/docs/use-cases/Avionics Engineer.md b/docs/use-cases/Avionics Engineer.md new file mode 100644 index 00000000..81ee1d68 --- /dev/null +++ b/docs/use-cases/Avionics Engineer.md @@ -0,0 +1,19 @@ + +## Cognix use case: the aviation engineer + +Let's take the example of an **Avionics Engineer** specialized in Fly-by-Wire (FBW) systems. + +The engineer receives a notification about a malfunction in the Fly-by-Wire system of an Airbus A320. The message indicates issues with the aircraft's electronic flight control system, which has led to instability in pitch and roll commands. + +2. **Preparing for the Task**: + - **Retrieve Documentation**: The engineer uses CogniX to quickly search for relevant sections in the Aircraft Maintenance Manual (AMM) and the specific service bulletins related to FBW systems. + - **Access Personal Notes**: CogniX retrieves the engineer’s previous troubleshooting notes on similar issues, highlighting successful resolutions and common pitfalls. + - **Simulate the Issue**: The engineer can also use CogniX to run simulations based on the error codes received from the aircraft’s Built-In Test Equipment (BITE), identifying potential root causes before heading to the aircraft. + +4. **Verification and Reporting**: + - **System Verification**: Post-repair, the engineer runs verification tests to ensure the FBW system is functioning correctly. CogniX logs the results and compares them with baseline data. + - **Reporting**: Finally, the engineer uses CogniX to generate a detailed report of the issue, the steps taken to resolve it, and any follow-up actions required. This report is automatically synced with the maintenance database for future reference. + +--- + +With CogniX, the avionics engineer saves significant time by having immediate access to necessary manuals, notes, and diagnostic tools, enabling a more efficient and accurate repair process. \ No newline at end of file diff --git a/license b/license new file mode 100644 index 00000000..ebba4cb5 --- /dev/null +++ b/license @@ -0,0 +1,37 @@ +This project is dual-licensed: + +## **Non-Commercial Use**: +Licensed under the Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0). See the [LICENSE](LICENSE) file for details. + +Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0) + +You are free to: +- Share — copy and redistribute the material in any medium or format +- Adapt — remix, transform, and build upon the material + +Under the following terms: +- Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. +- NonCommercial — You may not use the material for commercial purposes. + +No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits. + +View the full license at https://creativecommons.org/licenses/by-nc/4.0/ + + +## **Commercial Use**: +CogniX Commercial License + +Copyright [2024] [Generative Mind GmbH] + +This license allows the Licensee to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, provided that the following conditions are met: + +1. Commercial use of the Software requires explicit permission from the Licensor. +2. The Licensee must contact the Licensor to negotiate terms, which may include fees, royalties, or other conditions as agreed upon by both parties. +3. The Licensee must include the following acknowledgment in any public distribution of the Software: "This product includes software developed by [Generative Mind GmbH]." + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For commercial use, please contact: +- [Cognix](https://www.cognix.ch) +- Gian Paolo Santopaolo gp@cognix.ch +41794505007 +- Michael Gorog mg@cognix.ch \ No newline at end of file diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 00000000..5c94cdf1 Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/assets/react.svg b/src/assets/react.svg deleted file mode 100644 index 6c87de9b..00000000 --- a/src/assets/react.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/src/backend/.DS_Store b/src/backend/.DS_Store new file mode 100644 index 00000000..35794212 Binary files /dev/null and b/src/backend/.DS_Store differ diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile new file mode 100644 index 00000000..5fedac54 --- /dev/null +++ b/src/backend/Dockerfile @@ -0,0 +1,26 @@ +# select image +FROM golang:1.22-alpine as builder + +WORKDIR /app + +COPY go.mod go.sum ./ + +RUN go mod download +ARG service +ENV SERVICE_NAME $service +COPY ./core ./core +COPY ./$service ./$service +RUN mkdir bin +RUN go mod vendor +RUN go build -o ./bin/main ./${SERVICE_NAME} + +# Command to run the executable +CMD ["./main"] + +# select image +FROM alpine +ARG service +WORKDIR /app +RUN apk update && apk add --no-cache tzdata ca-certificates +COPY --from=builder /app/bin/ ./ +CMD ["./main"] diff --git a/src/backend/Dockerfile.dev b/src/backend/Dockerfile.dev new file mode 100644 index 00000000..2581b2f0 --- /dev/null +++ b/src/backend/Dockerfile.dev @@ -0,0 +1,10 @@ +# Dockerfile + +FROM golang:1.22-alpine +ARG service +ENV SERVICE_NAME $service +WORKDIR /backend/${SERVICE_NAME} +COPY ../go.mod ../go.sum ../ +RUN go install github.com/cosmtrek/air@latest +RUN go mod download +CMD ["air", "-d"] diff --git a/src/backend/Dockerfile.prod b/src/backend/Dockerfile.prod new file mode 100644 index 00000000..5fedac54 --- /dev/null +++ b/src/backend/Dockerfile.prod @@ -0,0 +1,26 @@ +# select image +FROM golang:1.22-alpine as builder + +WORKDIR /app + +COPY go.mod go.sum ./ + +RUN go mod download +ARG service +ENV SERVICE_NAME $service +COPY ./core ./core +COPY ./$service ./$service +RUN mkdir bin +RUN go mod vendor +RUN go build -o ./bin/main ./${SERVICE_NAME} + +# Command to run the executable +CMD ["./main"] + +# select image +FROM alpine +ARG service +WORKDIR /app +RUN apk update && apk add --no-cache tzdata ca-certificates +COPY --from=builder /app/bin/ ./ +CMD ["./main"] diff --git a/src/backend/Makefile b/src/backend/Makefile new file mode 100644 index 00000000..6432542a --- /dev/null +++ b/src/backend/Makefile @@ -0,0 +1,9 @@ +lint: + golangci-lint run --timeout=10m -v ./... + + +swag-api: + swag init --dir ./api --output ./api/docs --parseDependency --parseInternal + +gen-proto: + protoc -I=. -I=vendor -I=${GOPATH}/src core/proto/*.proto --go_out=. --go-grpc_out=. \ No newline at end of file diff --git a/src/backend/api/api-service-deployment.yaml b/src/backend/api/api-service-deployment.yaml new file mode 100644 index 00000000..4518ca7b --- /dev/null +++ b/src/backend/api/api-service-deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: apiservice +spec: + replicas: 1 + selector: + matchLabels: + app: apiservice + template: + metadata: + labels: + app: apiservice + spec: + containers: + - name: apiservice + image: cognixacr.azurecr.io/cognix/apiservice:main + envFrom: + - configMapRef: + name: api-srv + - configMapRef: + name: cockroach-cli + - configMapRef: + name: minio-cli + - configMapRef: + name: milvus-cli + - configMapRef: + name: nats-cli + - configMapRef: + name: embedder-cli + - configMapRef: + name: env-configmap + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: /backend + name: storage-volume + imagePullPolicy: Always + imagePullSecrets: + - name: pullsecret + volumes: + - name: storage-volume + persistentVolumeClaim: + claimName: storage-volume +--- +apiVersion: v1 +kind: Service +metadata: + name: api +spec: + selector: + app: apiservice + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-volume + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: azurefile-csi \ No newline at end of file diff --git a/src/backend/api/docs/docs.go b/src/backend/api/docs/docs.go new file mode 100644 index 00000000..52dcd17c --- /dev/null +++ b/src/backend/api/docs/docs.go @@ -0,0 +1,1696 @@ +// Package docs GENERATED BY SWAG; DO NOT EDIT +// This file was generated by swaggo/swag +package docs + +import "github.com/swaggo/swag" + +const docTemplate = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": { + "name": "API Support" + }, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/auth/google/login": { + "get": { + "description": "login using google auth", + "produces": [ + "application/json" + ], + "tags": [ + "Auth" + ], + "summary": "login using google auth", + "operationId": "auth_login", + "parameters": [ + { + "type": "string", + "description": "redirect base url", + "name": "redirect_url", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, + "/chats/create-chat-session": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "creates new chat session", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "creates new chat session", + "operationId": "chat_create_session", + "parameters": [ + { + "description": "create session parameters", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.CreateChatSession" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatSession" + } + } + } + } + }, + "/chats/get-chat-session/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return chat session with messages by given id", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "return chat session with messages by given id", + "operationId": "chat_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "session id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatSession" + } + } + } + } + }, + "/chats/get-user-chat-sessions": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of chat sessions for current user", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "return list of chat sessions for current user", + "operationId": "chat_get_sessions", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.ChatSession" + } + } + } + } + } + }, + "/chats/message/feedback": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "add feedback to message", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "add feedback to message", + "operationId": "chat_message_feedback", + "parameters": [ + { + "description": "send message parameters", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.MessageFeedbackParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatMessageFeedback" + } + } + } + } + }, + "/chats/send-message": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "send message and wait stream response", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "send message and wait stream response", + "operationId": "chat_send_message", + "parameters": [ + { + "description": "send message parameters", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.CreateChatMessageRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatMessage" + } + } + } + } + }, + "/manage/connector": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of allowed connectors", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "return list of allowed connectors", + "operationId": "connectors_get_all", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + } + }, + "/manage/connector/": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "creates connector", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "creates connector", + "operationId": "connectors_create", + "parameters": [ + { + "description": "connector create parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.CreateConnectorParam" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + }, + "/manage/connector/source_types": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of source types", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "return list of source types", + "operationId": "connectors_get_source_types", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.SourceTypeDescription" + } + } + } + } + } + }, + "/manage/connector/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of allowed connectors", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "return list of allowed connectors", + "operationId": "connectors_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "connector id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "updates connector", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "updates connector", + "operationId": "connectors_update", + "parameters": [ + { + "type": "integer", + "description": "connector id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "connector update parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.UpdateConnectorParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + }, + "/manage/connector/{id}/{action}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete or restore connector", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "delete or restore connector", + "operationId": "Connectors_delete_restore", + "parameters": [ + { + "type": "integer", + "description": "Connectors id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "action : delete | restore ", + "name": "action", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + }, + "/manage/embedding_models": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of embedding models", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "return list of embedding models", + "operationId": "embedding_model_get_all", + "parameters": [ + { + "type": "boolean", + "description": "true for include deleted embedding models", + "name": "archived", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "creates embedding models", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "creates embedding models", + "operationId": "embedding_model_create", + "parameters": [ + { + "description": "embedding model parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.EmbeddingModelParam" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "/manage/embedding_models/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return embedding model by id", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "return embedding model by id", + "operationId": "embedding_model_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "embedding model id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "updates embedding model", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "updates embedding model", + "operationId": "embedding_model_update", + "parameters": [ + { + "type": "integer", + "description": "embedding model id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "embedding model parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.EmbeddingModelParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "/manage/embedding_models/{id}/{action}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete or restore embedding model", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "delete or restore embedding model", + "operationId": "embedding_model_delete", + "parameters": [ + { + "type": "integer", + "description": "embedding model id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "action : delete | restore ", + "name": "action", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "/manage/personas": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of allowed personas", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "return list of allowed personas", + "operationId": "personas_get_all", + "parameters": [ + { + "type": "boolean", + "description": "true for include deleted personas", + "name": "archived", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "create persona", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "create persona", + "operationId": "persona_create", + "parameters": [ + { + "description": "persona payload", + "name": "id", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.PersonaParam" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "/manage/personas/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return persona by id", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "return persona by id", + "operationId": "persona_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "persona id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "update persona", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "update persona", + "operationId": "persona_update", + "parameters": [ + { + "type": "integer", + "description": "persona id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "persona payload", + "name": "id", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.PersonaParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "/manage/personas/{id}/{action}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete or restore persona", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "delete or restore persona", + "operationId": "persona_delete_restore", + "parameters": [ + { + "type": "integer", + "description": "persona id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "action : delete | restore ", + "name": "action", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "/tenant/user_info": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "get user info", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "get user info", + "operationId": "tenant_get_user_info", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.User" + } + } + } + } + }, + "/tenant/users": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of users", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "return list of users", + "operationId": "tenant_get_users", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.User" + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "add new user", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "add new user", + "operationId": "tenant_add_user", + "parameters": [ + { + "description": "create user parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.AddUserParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.User" + } + } + } + } + }, + "/tenant/users/{id}": { + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "edit user", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "edit user", + "operationId": "tenant_edit_user", + "parameters": [ + { + "type": "string", + "description": "user id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "edit user parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.EditUserParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.User" + } + } + } + } + } + }, + "definitions": { + "decimal.NullDecimal": { + "type": "object", + "properties": { + "decimal": { + "type": "number" + }, + "valid": { + "type": "boolean" + } + } + }, + "model.ChatMessage": { + "type": "object", + "properties": { + "chat_session_id": { + "type": "number" + }, + "citations": { + "type": "array", + "items": { + "$ref": "#/definitions/model.DocumentResponse" + } + }, + "error": { + "type": "string" + }, + "feedback": { + "$ref": "#/definitions/model.ChatMessageFeedback" + }, + "id": { + "type": "number" + }, + "latest_child_message": { + "type": "integer" + }, + "message": { + "type": "string" + }, + "message_type": { + "type": "string" + }, + "parent_message": { + "type": "number" + }, + "rephrased_query": { + "type": "string" + }, + "time_sent": { + "type": "string" + }, + "token_count": { + "type": "integer" + } + } + }, + "model.ChatMessageFeedback": { + "type": "object", + "properties": { + "chat_message_id": { + "type": "number" + }, + "feedback": { + "type": "string" + }, + "id": { + "type": "number" + }, + "up_votes": { + "type": "boolean" + }, + "user_id": { + "type": "string" + } + } + }, + "model.ChatSession": { + "type": "object", + "properties": { + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "description": { + "type": "string" + }, + "id": { + "type": "number" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/model.ChatMessage" + } + }, + "one_shot": { + "type": "boolean" + }, + "persona": { + "$ref": "#/definitions/model.Persona" + }, + "persona_id": { + "type": "number" + }, + "user_id": { + "type": "string" + } + } + }, + "model.Connector": { + "type": "object", + "properties": { + "connector_specific_config": { + "$ref": "#/definitions/model.JSONMap" + }, + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "docs": { + "type": "array", + "items": { + "$ref": "#/definitions/model.Document" + } + }, + "docs_map": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/model.Document" + } + }, + "id": { + "type": "number" + }, + "last_successful_analysis": { + "$ref": "#/definitions/pg.NullTime" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "name": { + "type": "string" + }, + "refresh_freq": { + "type": "integer" + }, + "source": { + "$ref": "#/definitions/model.SourceType" + }, + "status": { + "type": "string" + }, + "tenant_id": { + "$ref": "#/definitions/uuid.NullUUID" + }, + "total_docs_indexed": { + "type": "integer" + }, + "user_id": { + "type": "string" + } + } + }, + "model.Document": { + "type": "object", + "properties": { + "analyzed": { + "type": "boolean" + }, + "chunking_session": { + "$ref": "#/definitions/uuid.NullUUID" + }, + "connector_id": { + "type": "number" + }, + "creation_date": { + "type": "string" + }, + "id": { + "type": "number" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "parent_id": { + "$ref": "#/definitions/decimal.NullDecimal" + }, + "signature": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "model.DocumentResponse": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "document_id": { + "type": "string" + }, + "id": { + "type": "number" + }, + "link": { + "type": "string" + }, + "message_id": { + "type": "number" + }, + "updated_date": { + "type": "string" + } + } + }, + "model.EmbeddingModel": { + "type": "object", + "properties": { + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "id": { + "type": "number" + }, + "is_active": { + "type": "boolean" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "model_dim": { + "type": "integer" + }, + "model_id": { + "type": "string" + }, + "model_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "model.JSONMap": { + "type": "object", + "additionalProperties": true + }, + "model.LLM": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "endpoint": { + "type": "string" + }, + "id": { + "type": "number" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "model_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "model.Persona": { + "type": "object", + "properties": { + "chat_sessions": { + "type": "array", + "items": { + "$ref": "#/definitions/model.ChatSession" + } + }, + "creation_date": { + "type": "string" + }, + "default_persona": { + "type": "boolean" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "description": { + "type": "string" + }, + "display_priority": { + "type": "integer" + }, + "id": { + "type": "number" + }, + "is_visible": { + "type": "boolean" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "llm": { + "$ref": "#/definitions/model.LLM" + }, + "llm_id": { + "type": "number" + }, + "name": { + "type": "string" + }, + "prompt": { + "$ref": "#/definitions/model.Prompt" + }, + "starter_messages": { + "type": "array", + "items": { + "type": "integer" + } + }, + "tenant_id": { + "type": "string" + } + } + }, + "model.Prompt": { + "type": "object", + "properties": { + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "description": { + "type": "string" + }, + "id": { + "type": "number" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "name": { + "type": "string" + }, + "persona_id": { + "type": "number" + }, + "system_prompt": { + "type": "string" + }, + "task_prompt": { + "type": "string" + }, + "user_id": { + "type": "string" + } + } + }, + "model.SourceType": { + "type": "string", + "enum": [ + "ingestion_api", + "slack", + "web", + "google_drive", + "gmail", + "requesttracker", + "github", + "gitlab", + "guru", + "bookstack", + "confluence", + "slab", + "jira", + "productboard", + "file", + "notion", + "zulip", + "linear", + "hubspot", + "document360", + "gong", + "google_sites", + "zendesk", + "loopio", + "sharepoint", + "one-drive", + "msteams", + "youtube" + ], + "x-enum-varnames": [ + "SourceTypeIngestionApi", + "SourceTypeSlack", + "SourceTypeWEB", + "SourceTypeGoogleDrive", + "SourceTypeGMAIL", + "SourceTypeRequesttracker", + "SourceTypeGithub", + "SourceTypeGitlab", + "SourceTypeGuru", + "SourceTypeBookstack", + "SourceTypeConfluence", + "SourceTypeSlab", + "SourceTypeJira", + "SourceTypeProductboard", + "SourceTypeFile", + "SourceTypeNotion", + "SourceTypeZulip", + "SourceTypeLinear", + "SourceTypeHubspot", + "SourceTypeDocument360", + "SourceTypeGong", + "SourceTypeGoogleSites", + "SourceTypeZendesk", + "SourceTypeLoopio", + "SourceTypeSharepoint", + "SourceTypeOneDrive", + "SourceTypeMsTeams", + "SourceTypeYoutube" + ] + }, + "model.SourceTypeDescription": { + "type": "object", + "properties": { + "id": { + "$ref": "#/definitions/model.SourceType" + }, + "isImplemented": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "model.Tenant": { + "type": "object", + "properties": { + "configuration": { + "$ref": "#/definitions/model.JSONMap" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "model.User": { + "type": "object", + "properties": { + "first_name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "roles": { + "type": "array", + "items": { + "type": "string" + } + }, + "tenant": { + "$ref": "#/definitions/model.Tenant" + }, + "tenant_id": { + "type": "string" + }, + "user_name": { + "type": "string" + } + } + }, + "parameters.AddUserParam": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "parameters.CreateChatMessageRequest": { + "type": "object", + "properties": { + "chat_session_id": { + "type": "number" + }, + "message": { + "type": "string" + }, + "no_ai_answer": { + "type": "boolean" + }, + "parent_message_id": { + "type": "number" + }, + "prompt_id": { + "type": "number" + }, + "query_override": { + "description": "RetrievalOptions RetrievalDetails ` + "`" + `json:\"retrieval_options,omitempty\"` + "`" + `", + "type": "string" + }, + "search_doc_ids": { + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "parameters.CreateChatSession": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "one_shot": { + "type": "boolean" + }, + "persona_id": { + "type": "number" + } + } + }, + "parameters.CreateConnectorParam": { + "type": "object", + "properties": { + "connector_specific_config": { + "$ref": "#/definitions/model.JSONMap" + }, + "disabled": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "refresh_freq": { + "type": "integer" + }, + "shared": { + "type": "boolean" + }, + "source": { + "type": "string" + } + } + }, + "parameters.EditUserParam": { + "type": "object", + "properties": { + "role": { + "type": "string" + } + } + }, + "parameters.EmbeddingModelParam": { + "type": "object", + "properties": { + "is_active": { + "type": "boolean" + }, + "model_dim": { + "type": "integer" + }, + "model_id": { + "type": "string" + }, + "model_name": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "parameters.MessageFeedbackParam": { + "type": "object", + "properties": { + "id": { + "type": "number" + }, + "vote": { + "type": "string" + } + } + }, + "parameters.PersonaParam": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "description": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "model_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "starter_messages": { + "type": "array", + "items": { + "$ref": "#/definitions/parameters.StarterMessage" + } + }, + "system_prompt": { + "type": "string" + }, + "task_prompt": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "parameters.StarterMessage": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "message": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "parameters.UpdateConnectorParam": { + "type": "object", + "properties": { + "connector_specific_config": { + "$ref": "#/definitions/model.JSONMap" + }, + "name": { + "type": "string" + }, + "refresh_freq": { + "type": "integer" + }, + "shared": { + "type": "boolean" + }, + "status": { + "type": "string" + } + } + }, + "pg.NullTime": { + "type": "object", + "properties": { + "time.Time": { + "type": "string" + } + } + }, + "uuid.NullUUID": { + "type": "object", + "properties": { + "uuid": { + "type": "string" + }, + "valid": { + "description": "Valid is true if UUID is not NULL", + "type": "boolean" + } + } + } + }, + "securityDefinitions": { + "ApiKeyAuth": { + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + } +}` + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = &swag.Spec{ + Version: "1.0", + Host: "", + BasePath: "/api", + Schemes: []string{}, + Title: "Cognix API", + Description: "This is Cognix Golang API Documentation", + InfoInstanceName: "swagger", + SwaggerTemplate: docTemplate, +} + +func init() { + swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) +} diff --git a/src/backend/api/docs/swagger.json b/src/backend/api/docs/swagger.json new file mode 100644 index 00000000..671750ef --- /dev/null +++ b/src/backend/api/docs/swagger.json @@ -0,0 +1,1672 @@ +{ + "swagger": "2.0", + "info": { + "description": "This is Cognix Golang API Documentation", + "title": "Cognix API", + "contact": { + "name": "API Support" + }, + "version": "1.0" + }, + "basePath": "/api", + "paths": { + "/auth/google/login": { + "get": { + "description": "login using google auth", + "produces": [ + "application/json" + ], + "tags": [ + "Auth" + ], + "summary": "login using google auth", + "operationId": "auth_login", + "parameters": [ + { + "type": "string", + "description": "redirect base url", + "name": "redirect_url", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, + "/chats/create-chat-session": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "creates new chat session", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "creates new chat session", + "operationId": "chat_create_session", + "parameters": [ + { + "description": "create session parameters", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.CreateChatSession" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatSession" + } + } + } + } + }, + "/chats/get-chat-session/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return chat session with messages by given id", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "return chat session with messages by given id", + "operationId": "chat_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "session id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatSession" + } + } + } + } + }, + "/chats/get-user-chat-sessions": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of chat sessions for current user", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "return list of chat sessions for current user", + "operationId": "chat_get_sessions", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.ChatSession" + } + } + } + } + } + }, + "/chats/message/feedback": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "add feedback to message", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "add feedback to message", + "operationId": "chat_message_feedback", + "parameters": [ + { + "description": "send message parameters", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.MessageFeedbackParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatMessageFeedback" + } + } + } + } + }, + "/chats/send-message": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "send message and wait stream response", + "produces": [ + "application/json" + ], + "tags": [ + "Chat" + ], + "summary": "send message and wait stream response", + "operationId": "chat_send_message", + "parameters": [ + { + "description": "send message parameters", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.CreateChatMessageRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.ChatMessage" + } + } + } + } + }, + "/manage/connector": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of allowed connectors", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "return list of allowed connectors", + "operationId": "connectors_get_all", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + } + }, + "/manage/connector/": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "creates connector", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "creates connector", + "operationId": "connectors_create", + "parameters": [ + { + "description": "connector create parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.CreateConnectorParam" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + }, + "/manage/connector/source_types": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of source types", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "return list of source types", + "operationId": "connectors_get_source_types", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.SourceTypeDescription" + } + } + } + } + } + }, + "/manage/connector/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of allowed connectors", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "return list of allowed connectors", + "operationId": "connectors_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "connector id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "updates connector", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "updates connector", + "operationId": "connectors_update", + "parameters": [ + { + "type": "integer", + "description": "connector id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "connector update parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.UpdateConnectorParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + }, + "/manage/connector/{id}/{action}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete or restore connector", + "produces": [ + "application/json" + ], + "tags": [ + "Connectors" + ], + "summary": "delete or restore connector", + "operationId": "Connectors_delete_restore", + "parameters": [ + { + "type": "integer", + "description": "Connectors id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "action : delete | restore ", + "name": "action", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Connector" + } + } + } + } + }, + "/manage/embedding_models": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of embedding models", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "return list of embedding models", + "operationId": "embedding_model_get_all", + "parameters": [ + { + "type": "boolean", + "description": "true for include deleted embedding models", + "name": "archived", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "creates embedding models", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "creates embedding models", + "operationId": "embedding_model_create", + "parameters": [ + { + "description": "embedding model parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.EmbeddingModelParam" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "/manage/embedding_models/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return embedding model by id", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "return embedding model by id", + "operationId": "embedding_model_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "embedding model id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "updates embedding model", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "updates embedding model", + "operationId": "embedding_model_update", + "parameters": [ + { + "type": "integer", + "description": "embedding model id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "embedding model parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.EmbeddingModelParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "/manage/embedding_models/{id}/{action}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete or restore embedding model", + "produces": [ + "application/json" + ], + "tags": [ + "EmbeddingModel" + ], + "summary": "delete or restore embedding model", + "operationId": "embedding_model_delete", + "parameters": [ + { + "type": "integer", + "description": "embedding model id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "action : delete | restore ", + "name": "action", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.EmbeddingModel" + } + } + } + } + }, + "/manage/personas": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of allowed personas", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "return list of allowed personas", + "operationId": "personas_get_all", + "parameters": [ + { + "type": "boolean", + "description": "true for include deleted personas", + "name": "archived", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "create persona", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "create persona", + "operationId": "persona_create", + "parameters": [ + { + "description": "persona payload", + "name": "id", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.PersonaParam" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "/manage/personas/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return persona by id", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "return persona by id", + "operationId": "persona_get_by_id", + "parameters": [ + { + "type": "integer", + "description": "persona id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "update persona", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "update persona", + "operationId": "persona_update", + "parameters": [ + { + "type": "integer", + "description": "persona id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "persona payload", + "name": "id", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.PersonaParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "/manage/personas/{id}/{action}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete or restore persona", + "produces": [ + "application/json" + ], + "tags": [ + "Persona" + ], + "summary": "delete or restore persona", + "operationId": "persona_delete_restore", + "parameters": [ + { + "type": "integer", + "description": "persona id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "action : delete | restore ", + "name": "action", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.Persona" + } + } + } + } + }, + "/tenant/user_info": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "get user info", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "get user info", + "operationId": "tenant_get_user_info", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.User" + } + } + } + } + }, + "/tenant/users": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "return list of users", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "return list of users", + "operationId": "tenant_get_users", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/model.User" + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "add new user", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "add new user", + "operationId": "tenant_add_user", + "parameters": [ + { + "description": "create user parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.AddUserParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.User" + } + } + } + } + }, + "/tenant/users/{id}": { + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "edit user", + "produces": [ + "application/json" + ], + "tags": [ + "Tenant" + ], + "summary": "edit user", + "operationId": "tenant_edit_user", + "parameters": [ + { + "type": "string", + "description": "user id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "edit user parameter", + "name": "params", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/parameters.EditUserParam" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/model.User" + } + } + } + } + } + }, + "definitions": { + "decimal.NullDecimal": { + "type": "object", + "properties": { + "decimal": { + "type": "number" + }, + "valid": { + "type": "boolean" + } + } + }, + "model.ChatMessage": { + "type": "object", + "properties": { + "chat_session_id": { + "type": "number" + }, + "citations": { + "type": "array", + "items": { + "$ref": "#/definitions/model.DocumentResponse" + } + }, + "error": { + "type": "string" + }, + "feedback": { + "$ref": "#/definitions/model.ChatMessageFeedback" + }, + "id": { + "type": "number" + }, + "latest_child_message": { + "type": "integer" + }, + "message": { + "type": "string" + }, + "message_type": { + "type": "string" + }, + "parent_message": { + "type": "number" + }, + "rephrased_query": { + "type": "string" + }, + "time_sent": { + "type": "string" + }, + "token_count": { + "type": "integer" + } + } + }, + "model.ChatMessageFeedback": { + "type": "object", + "properties": { + "chat_message_id": { + "type": "number" + }, + "feedback": { + "type": "string" + }, + "id": { + "type": "number" + }, + "up_votes": { + "type": "boolean" + }, + "user_id": { + "type": "string" + } + } + }, + "model.ChatSession": { + "type": "object", + "properties": { + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "description": { + "type": "string" + }, + "id": { + "type": "number" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/model.ChatMessage" + } + }, + "one_shot": { + "type": "boolean" + }, + "persona": { + "$ref": "#/definitions/model.Persona" + }, + "persona_id": { + "type": "number" + }, + "user_id": { + "type": "string" + } + } + }, + "model.Connector": { + "type": "object", + "properties": { + "connector_specific_config": { + "$ref": "#/definitions/model.JSONMap" + }, + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "docs": { + "type": "array", + "items": { + "$ref": "#/definitions/model.Document" + } + }, + "docs_map": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/model.Document" + } + }, + "id": { + "type": "number" + }, + "last_successful_analysis": { + "$ref": "#/definitions/pg.NullTime" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "name": { + "type": "string" + }, + "refresh_freq": { + "type": "integer" + }, + "source": { + "$ref": "#/definitions/model.SourceType" + }, + "status": { + "type": "string" + }, + "tenant_id": { + "$ref": "#/definitions/uuid.NullUUID" + }, + "total_docs_indexed": { + "type": "integer" + }, + "user_id": { + "type": "string" + } + } + }, + "model.Document": { + "type": "object", + "properties": { + "analyzed": { + "type": "boolean" + }, + "chunking_session": { + "$ref": "#/definitions/uuid.NullUUID" + }, + "connector_id": { + "type": "number" + }, + "creation_date": { + "type": "string" + }, + "id": { + "type": "number" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "parent_id": { + "$ref": "#/definitions/decimal.NullDecimal" + }, + "signature": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "model.DocumentResponse": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "document_id": { + "type": "string" + }, + "id": { + "type": "number" + }, + "link": { + "type": "string" + }, + "message_id": { + "type": "number" + }, + "updated_date": { + "type": "string" + } + } + }, + "model.EmbeddingModel": { + "type": "object", + "properties": { + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "id": { + "type": "number" + }, + "is_active": { + "type": "boolean" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "model_dim": { + "type": "integer" + }, + "model_id": { + "type": "string" + }, + "model_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "model.JSONMap": { + "type": "object", + "additionalProperties": true + }, + "model.LLM": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "endpoint": { + "type": "string" + }, + "id": { + "type": "number" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "model_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "model.Persona": { + "type": "object", + "properties": { + "chat_sessions": { + "type": "array", + "items": { + "$ref": "#/definitions/model.ChatSession" + } + }, + "creation_date": { + "type": "string" + }, + "default_persona": { + "type": "boolean" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "description": { + "type": "string" + }, + "display_priority": { + "type": "integer" + }, + "id": { + "type": "number" + }, + "is_visible": { + "type": "boolean" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "llm": { + "$ref": "#/definitions/model.LLM" + }, + "llm_id": { + "type": "number" + }, + "name": { + "type": "string" + }, + "prompt": { + "$ref": "#/definitions/model.Prompt" + }, + "starter_messages": { + "type": "array", + "items": { + "type": "integer" + } + }, + "tenant_id": { + "type": "string" + } + } + }, + "model.Prompt": { + "type": "object", + "properties": { + "creation_date": { + "type": "string" + }, + "deleted_date": { + "$ref": "#/definitions/pg.NullTime" + }, + "description": { + "type": "string" + }, + "id": { + "type": "number" + }, + "last_update": { + "$ref": "#/definitions/pg.NullTime" + }, + "name": { + "type": "string" + }, + "persona_id": { + "type": "number" + }, + "system_prompt": { + "type": "string" + }, + "task_prompt": { + "type": "string" + }, + "user_id": { + "type": "string" + } + } + }, + "model.SourceType": { + "type": "string", + "enum": [ + "ingestion_api", + "slack", + "web", + "google_drive", + "gmail", + "requesttracker", + "github", + "gitlab", + "guru", + "bookstack", + "confluence", + "slab", + "jira", + "productboard", + "file", + "notion", + "zulip", + "linear", + "hubspot", + "document360", + "gong", + "google_sites", + "zendesk", + "loopio", + "sharepoint", + "one-drive", + "msteams", + "youtube" + ], + "x-enum-varnames": [ + "SourceTypeIngestionApi", + "SourceTypeSlack", + "SourceTypeWEB", + "SourceTypeGoogleDrive", + "SourceTypeGMAIL", + "SourceTypeRequesttracker", + "SourceTypeGithub", + "SourceTypeGitlab", + "SourceTypeGuru", + "SourceTypeBookstack", + "SourceTypeConfluence", + "SourceTypeSlab", + "SourceTypeJira", + "SourceTypeProductboard", + "SourceTypeFile", + "SourceTypeNotion", + "SourceTypeZulip", + "SourceTypeLinear", + "SourceTypeHubspot", + "SourceTypeDocument360", + "SourceTypeGong", + "SourceTypeGoogleSites", + "SourceTypeZendesk", + "SourceTypeLoopio", + "SourceTypeSharepoint", + "SourceTypeOneDrive", + "SourceTypeMsTeams", + "SourceTypeYoutube" + ] + }, + "model.SourceTypeDescription": { + "type": "object", + "properties": { + "id": { + "$ref": "#/definitions/model.SourceType" + }, + "isImplemented": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "model.Tenant": { + "type": "object", + "properties": { + "configuration": { + "$ref": "#/definitions/model.JSONMap" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "model.User": { + "type": "object", + "properties": { + "first_name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "roles": { + "type": "array", + "items": { + "type": "string" + } + }, + "tenant": { + "$ref": "#/definitions/model.Tenant" + }, + "tenant_id": { + "type": "string" + }, + "user_name": { + "type": "string" + } + } + }, + "parameters.AddUserParam": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "parameters.CreateChatMessageRequest": { + "type": "object", + "properties": { + "chat_session_id": { + "type": "number" + }, + "message": { + "type": "string" + }, + "no_ai_answer": { + "type": "boolean" + }, + "parent_message_id": { + "type": "number" + }, + "prompt_id": { + "type": "number" + }, + "query_override": { + "description": "RetrievalOptions RetrievalDetails `json:\"retrieval_options,omitempty\"`", + "type": "string" + }, + "search_doc_ids": { + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "parameters.CreateChatSession": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "one_shot": { + "type": "boolean" + }, + "persona_id": { + "type": "number" + } + } + }, + "parameters.CreateConnectorParam": { + "type": "object", + "properties": { + "connector_specific_config": { + "$ref": "#/definitions/model.JSONMap" + }, + "disabled": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "refresh_freq": { + "type": "integer" + }, + "shared": { + "type": "boolean" + }, + "source": { + "type": "string" + } + } + }, + "parameters.EditUserParam": { + "type": "object", + "properties": { + "role": { + "type": "string" + } + } + }, + "parameters.EmbeddingModelParam": { + "type": "object", + "properties": { + "is_active": { + "type": "boolean" + }, + "model_dim": { + "type": "integer" + }, + "model_id": { + "type": "string" + }, + "model_name": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "parameters.MessageFeedbackParam": { + "type": "object", + "properties": { + "id": { + "type": "number" + }, + "vote": { + "type": "string" + } + } + }, + "parameters.PersonaParam": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "description": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "model_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "starter_messages": { + "type": "array", + "items": { + "$ref": "#/definitions/parameters.StarterMessage" + } + }, + "system_prompt": { + "type": "string" + }, + "task_prompt": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "parameters.StarterMessage": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "message": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "parameters.UpdateConnectorParam": { + "type": "object", + "properties": { + "connector_specific_config": { + "$ref": "#/definitions/model.JSONMap" + }, + "name": { + "type": "string" + }, + "refresh_freq": { + "type": "integer" + }, + "shared": { + "type": "boolean" + }, + "status": { + "type": "string" + } + } + }, + "pg.NullTime": { + "type": "object", + "properties": { + "time.Time": { + "type": "string" + } + } + }, + "uuid.NullUUID": { + "type": "object", + "properties": { + "uuid": { + "type": "string" + }, + "valid": { + "description": "Valid is true if UUID is not NULL", + "type": "boolean" + } + } + } + }, + "securityDefinitions": { + "ApiKeyAuth": { + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + } +} \ No newline at end of file diff --git a/src/backend/api/docs/swagger.yaml b/src/backend/api/docs/swagger.yaml new file mode 100644 index 00000000..0ae2fe6d --- /dev/null +++ b/src/backend/api/docs/swagger.yaml @@ -0,0 +1,1091 @@ +basePath: /api +definitions: + decimal.NullDecimal: + properties: + decimal: + type: number + valid: + type: boolean + type: object + model.ChatMessage: + properties: + chat_session_id: + type: number + citations: + items: + $ref: '#/definitions/model.DocumentResponse' + type: array + error: + type: string + feedback: + $ref: '#/definitions/model.ChatMessageFeedback' + id: + type: number + latest_child_message: + type: integer + message: + type: string + message_type: + type: string + parent_message: + type: number + rephrased_query: + type: string + time_sent: + type: string + token_count: + type: integer + type: object + model.ChatMessageFeedback: + properties: + chat_message_id: + type: number + feedback: + type: string + id: + type: number + up_votes: + type: boolean + user_id: + type: string + type: object + model.ChatSession: + properties: + creation_date: + type: string + deleted_date: + $ref: '#/definitions/pg.NullTime' + description: + type: string + id: + type: number + messages: + items: + $ref: '#/definitions/model.ChatMessage' + type: array + one_shot: + type: boolean + persona: + $ref: '#/definitions/model.Persona' + persona_id: + type: number + user_id: + type: string + type: object + model.Connector: + properties: + connector_specific_config: + $ref: '#/definitions/model.JSONMap' + creation_date: + type: string + deleted_date: + $ref: '#/definitions/pg.NullTime' + docs: + items: + $ref: '#/definitions/model.Document' + type: array + docs_map: + additionalProperties: + $ref: '#/definitions/model.Document' + type: object + id: + type: number + last_successful_analysis: + $ref: '#/definitions/pg.NullTime' + last_update: + $ref: '#/definitions/pg.NullTime' + name: + type: string + refresh_freq: + type: integer + source: + $ref: '#/definitions/model.SourceType' + status: + type: string + tenant_id: + $ref: '#/definitions/uuid.NullUUID' + total_docs_indexed: + type: integer + user_id: + type: string + type: object + model.Document: + properties: + analyzed: + type: boolean + chunking_session: + $ref: '#/definitions/uuid.NullUUID' + connector_id: + type: number + creation_date: + type: string + id: + type: number + last_update: + $ref: '#/definitions/pg.NullTime' + parent_id: + $ref: '#/definitions/decimal.NullDecimal' + signature: + type: string + source_id: + type: string + url: + type: string + type: object + model.DocumentResponse: + properties: + content: + type: string + document_id: + type: string + id: + type: number + link: + type: string + message_id: + type: number + updated_date: + type: string + type: object + model.EmbeddingModel: + properties: + creation_date: + type: string + deleted_date: + $ref: '#/definitions/pg.NullTime' + id: + type: number + is_active: + type: boolean + last_update: + $ref: '#/definitions/pg.NullTime' + model_dim: + type: integer + model_id: + type: string + model_name: + type: string + tenant_id: + type: string + url: + type: string + type: object + model.JSONMap: + additionalProperties: true + type: object + model.LLM: + properties: + api_key: + type: string + creation_date: + type: string + deleted_date: + $ref: '#/definitions/pg.NullTime' + endpoint: + type: string + id: + type: number + last_update: + $ref: '#/definitions/pg.NullTime' + model_id: + type: string + name: + type: string + tenant_id: + type: string + url: + type: string + type: object + model.Persona: + properties: + chat_sessions: + items: + $ref: '#/definitions/model.ChatSession' + type: array + creation_date: + type: string + default_persona: + type: boolean + deleted_date: + $ref: '#/definitions/pg.NullTime' + description: + type: string + display_priority: + type: integer + id: + type: number + is_visible: + type: boolean + last_update: + $ref: '#/definitions/pg.NullTime' + llm: + $ref: '#/definitions/model.LLM' + llm_id: + type: number + name: + type: string + prompt: + $ref: '#/definitions/model.Prompt' + starter_messages: + items: + type: integer + type: array + tenant_id: + type: string + type: object + model.Prompt: + properties: + creation_date: + type: string + deleted_date: + $ref: '#/definitions/pg.NullTime' + description: + type: string + id: + type: number + last_update: + $ref: '#/definitions/pg.NullTime' + name: + type: string + persona_id: + type: number + system_prompt: + type: string + task_prompt: + type: string + user_id: + type: string + type: object + model.SourceType: + enum: + - ingestion_api + - slack + - web + - google_drive + - gmail + - requesttracker + - github + - gitlab + - guru + - bookstack + - confluence + - slab + - jira + - productboard + - file + - notion + - zulip + - linear + - hubspot + - document360 + - gong + - google_sites + - zendesk + - loopio + - sharepoint + - one-drive + - msteams + - youtube + type: string + x-enum-varnames: + - SourceTypeIngestionApi + - SourceTypeSlack + - SourceTypeWEB + - SourceTypeGoogleDrive + - SourceTypeGMAIL + - SourceTypeRequesttracker + - SourceTypeGithub + - SourceTypeGitlab + - SourceTypeGuru + - SourceTypeBookstack + - SourceTypeConfluence + - SourceTypeSlab + - SourceTypeJira + - SourceTypeProductboard + - SourceTypeFile + - SourceTypeNotion + - SourceTypeZulip + - SourceTypeLinear + - SourceTypeHubspot + - SourceTypeDocument360 + - SourceTypeGong + - SourceTypeGoogleSites + - SourceTypeZendesk + - SourceTypeLoopio + - SourceTypeSharepoint + - SourceTypeOneDrive + - SourceTypeMsTeams + - SourceTypeYoutube + model.SourceTypeDescription: + properties: + id: + $ref: '#/definitions/model.SourceType' + isImplemented: + type: boolean + name: + type: string + type: object + model.Tenant: + properties: + configuration: + $ref: '#/definitions/model.JSONMap' + id: + type: string + name: + type: string + type: object + model.User: + properties: + first_name: + type: string + id: + type: string + last_name: + type: string + roles: + items: + type: string + type: array + tenant: + $ref: '#/definitions/model.Tenant' + tenant_id: + type: string + user_name: + type: string + type: object + parameters.AddUserParam: + properties: + email: + type: string + role: + type: string + type: object + parameters.CreateChatMessageRequest: + properties: + chat_session_id: + type: number + message: + type: string + no_ai_answer: + type: boolean + parent_message_id: + type: number + prompt_id: + type: number + query_override: + description: RetrievalOptions RetrievalDetails `json:"retrieval_options,omitempty"` + type: string + search_doc_ids: + items: + type: number + type: array + type: object + parameters.CreateChatSession: + properties: + description: + type: string + one_shot: + type: boolean + persona_id: + type: number + type: object + parameters.CreateConnectorParam: + properties: + connector_specific_config: + $ref: '#/definitions/model.JSONMap' + disabled: + type: boolean + name: + type: string + refresh_freq: + type: integer + shared: + type: boolean + source: + type: string + type: object + parameters.EditUserParam: + properties: + role: + type: string + type: object + parameters.EmbeddingModelParam: + properties: + is_active: + type: boolean + model_dim: + type: integer + model_id: + type: string + model_name: + type: string + url: + type: string + type: object + parameters.MessageFeedbackParam: + properties: + id: + type: number + vote: + type: string + type: object + parameters.PersonaParam: + properties: + api_key: + type: string + description: + type: string + endpoint: + type: string + model_id: + type: string + name: + type: string + starter_messages: + items: + $ref: '#/definitions/parameters.StarterMessage' + type: array + system_prompt: + type: string + task_prompt: + type: string + url: + type: string + type: object + parameters.StarterMessage: + properties: + description: + type: string + message: + type: string + name: + type: string + type: object + parameters.UpdateConnectorParam: + properties: + connector_specific_config: + $ref: '#/definitions/model.JSONMap' + name: + type: string + refresh_freq: + type: integer + shared: + type: boolean + status: + type: string + type: object + pg.NullTime: + properties: + time.Time: + type: string + type: object + uuid.NullUUID: + properties: + uuid: + type: string + valid: + description: Valid is true if UUID is not NULL + type: boolean + type: object +info: + contact: + name: API Support + description: This is Cognix Golang API Documentation + title: Cognix API + version: "1.0" +paths: + /auth/google/login: + get: + description: login using google auth + operationId: auth_login + parameters: + - description: redirect base url + in: query + name: redirect_url + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + type: string + summary: login using google auth + tags: + - Auth + /chats/create-chat-session: + post: + description: creates new chat session + operationId: chat_create_session + parameters: + - description: create session parameters + in: body + name: payload + required: true + schema: + $ref: '#/definitions/parameters.CreateChatSession' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.ChatSession' + security: + - ApiKeyAuth: [] + summary: creates new chat session + tags: + - Chat + /chats/get-chat-session/{id}: + get: + description: return chat session with messages by given id + operationId: chat_get_by_id + parameters: + - description: session id + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.ChatSession' + security: + - ApiKeyAuth: [] + summary: return chat session with messages by given id + tags: + - Chat + /chats/get-user-chat-sessions: + get: + description: return list of chat sessions for current user + operationId: chat_get_sessions + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/model.ChatSession' + type: array + security: + - ApiKeyAuth: [] + summary: return list of chat sessions for current user + tags: + - Chat + /chats/message/feedback: + post: + description: add feedback to message + operationId: chat_message_feedback + parameters: + - description: send message parameters + in: body + name: payload + required: true + schema: + $ref: '#/definitions/parameters.MessageFeedbackParam' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.ChatMessageFeedback' + security: + - ApiKeyAuth: [] + summary: add feedback to message + tags: + - Chat + /chats/send-message: + post: + description: send message and wait stream response + operationId: chat_send_message + parameters: + - description: send message parameters + in: body + name: payload + required: true + schema: + $ref: '#/definitions/parameters.CreateChatMessageRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.ChatMessage' + security: + - ApiKeyAuth: [] + summary: send message and wait stream response + tags: + - Chat + /manage/connector: + get: + description: return list of allowed connectors + operationId: connectors_get_all + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/model.Connector' + type: array + security: + - ApiKeyAuth: [] + summary: return list of allowed connectors + tags: + - Connectors + /manage/connector/: + post: + description: creates connector + operationId: connectors_create + parameters: + - description: connector create parameter + in: body + name: params + required: true + schema: + $ref: '#/definitions/parameters.CreateConnectorParam' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/model.Connector' + security: + - ApiKeyAuth: [] + summary: creates connector + tags: + - Connectors + /manage/connector/{id}: + get: + description: return list of allowed connectors + operationId: connectors_get_by_id + parameters: + - description: connector id + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.Connector' + security: + - ApiKeyAuth: [] + summary: return list of allowed connectors + tags: + - Connectors + put: + description: updates connector + operationId: connectors_update + parameters: + - description: connector id + in: path + name: id + required: true + type: integer + - description: connector update parameter + in: body + name: params + required: true + schema: + $ref: '#/definitions/parameters.UpdateConnectorParam' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.Connector' + security: + - ApiKeyAuth: [] + summary: updates connector + tags: + - Connectors + /manage/connector/{id}/{action}: + post: + description: delete or restore connector + operationId: Connectors_delete_restore + parameters: + - description: Connectors id + in: path + name: id + required: true + type: integer + - description: 'action : delete | restore ' + in: path + name: action + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.Connector' + security: + - ApiKeyAuth: [] + summary: delete or restore connector + tags: + - Connectors + /manage/connector/source_types: + get: + description: return list of source types + operationId: connectors_get_source_types + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/model.SourceTypeDescription' + type: array + security: + - ApiKeyAuth: [] + summary: return list of source types + tags: + - Connectors + /manage/embedding_models: + get: + description: return list of embedding models + operationId: embedding_model_get_all + parameters: + - description: true for include deleted embedding models + in: query + name: archived + type: boolean + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/model.EmbeddingModel' + type: array + security: + - ApiKeyAuth: [] + summary: return list of embedding models + tags: + - EmbeddingModel + post: + description: creates embedding models + operationId: embedding_model_create + parameters: + - description: embedding model parameter + in: body + name: params + required: true + schema: + $ref: '#/definitions/parameters.EmbeddingModelParam' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/model.EmbeddingModel' + security: + - ApiKeyAuth: [] + summary: creates embedding models + tags: + - EmbeddingModel + /manage/embedding_models/{id}: + get: + description: return embedding model by id + operationId: embedding_model_get_by_id + parameters: + - description: embedding model id + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.EmbeddingModel' + security: + - ApiKeyAuth: [] + summary: return embedding model by id + tags: + - EmbeddingModel + put: + description: updates embedding model + operationId: embedding_model_update + parameters: + - description: embedding model id + in: path + name: id + required: true + type: integer + - description: embedding model parameter + in: body + name: params + required: true + schema: + $ref: '#/definitions/parameters.EmbeddingModelParam' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.EmbeddingModel' + security: + - ApiKeyAuth: [] + summary: updates embedding model + tags: + - EmbeddingModel + /manage/embedding_models/{id}/{action}: + post: + description: delete or restore embedding model + operationId: embedding_model_delete + parameters: + - description: embedding model id + in: path + name: id + required: true + type: integer + - description: 'action : delete | restore ' + in: path + name: action + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.EmbeddingModel' + security: + - ApiKeyAuth: [] + summary: delete or restore embedding model + tags: + - EmbeddingModel + /manage/personas: + get: + description: return list of allowed personas + operationId: personas_get_all + parameters: + - description: true for include deleted personas + in: query + name: archived + type: boolean + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/model.Persona' + type: array + security: + - ApiKeyAuth: [] + summary: return list of allowed personas + tags: + - Persona + post: + description: create persona + operationId: persona_create + parameters: + - description: persona payload + in: body + name: id + required: true + schema: + $ref: '#/definitions/parameters.PersonaParam' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/model.Persona' + security: + - ApiKeyAuth: [] + summary: create persona + tags: + - Persona + /manage/personas/{id}: + get: + description: return persona by id + operationId: persona_get_by_id + parameters: + - description: persona id + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.Persona' + security: + - ApiKeyAuth: [] + summary: return persona by id + tags: + - Persona + put: + description: update persona + operationId: persona_update + parameters: + - description: persona id + in: path + name: id + required: true + type: integer + - description: persona payload + in: body + name: id + required: true + schema: + $ref: '#/definitions/parameters.PersonaParam' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.Persona' + security: + - ApiKeyAuth: [] + summary: update persona + tags: + - Persona + /manage/personas/{id}/{action}: + post: + description: delete or restore persona + operationId: persona_delete_restore + parameters: + - description: persona id + in: path + name: id + required: true + type: integer + - description: 'action : delete | restore ' + in: path + name: action + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.Persona' + security: + - ApiKeyAuth: [] + summary: delete or restore persona + tags: + - Persona + /tenant/user_info: + get: + description: get user info + operationId: tenant_get_user_info + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.User' + security: + - ApiKeyAuth: [] + summary: get user info + tags: + - Tenant + /tenant/users: + get: + description: return list of users + operationId: tenant_get_users + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/model.User' + type: array + security: + - ApiKeyAuth: [] + summary: return list of users + tags: + - Tenant + post: + description: add new user + operationId: tenant_add_user + parameters: + - description: create user parameter + in: body + name: params + required: true + schema: + $ref: '#/definitions/parameters.AddUserParam' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.User' + security: + - ApiKeyAuth: [] + summary: add new user + tags: + - Tenant + /tenant/users/{id}: + put: + description: edit user + operationId: tenant_edit_user + parameters: + - description: user id + in: path + name: id + required: true + type: string + - description: edit user parameter + in: body + name: params + required: true + schema: + $ref: '#/definitions/parameters.EditUserParam' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/model.User' + security: + - ApiKeyAuth: [] + summary: edit user + tags: + - Tenant +securityDefinitions: + ApiKeyAuth: + in: header + name: Authorization + type: apiKey +swagger: "2.0" diff --git a/src/backend/api/handler/auth.go b/src/backend/api/handler/auth.go new file mode 100644 index 00000000..3c3d46f0 --- /dev/null +++ b/src/backend/api/handler/auth.go @@ -0,0 +1,208 @@ +package handler + +import ( + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/oauth" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "encoding/base64" + "encoding/json" + "fmt" + "github.com/gin-gonic/gin" + "net/http" +) + +// AuthHandler handles authentication endpoints +type AuthHandler struct { + oauthClient oauth.Proxy + jwtService security.JWTService + authBL bll.AuthBL + //storage storage.Storage +} + +func NewAuthHandler(oauthClient oauth.Proxy, + jwtService security.JWTService, + authBL bll.AuthBL, + //storage storage.Storage, + +) *AuthHandler { + return &AuthHandler{oauthClient: oauthClient, + jwtService: jwtService, + authBL: authBL, + //storage: storage, + } +} + +func (h *AuthHandler) Mount(route *gin.Engine, authMiddleware gin.HandlerFunc) { + handler := route.Group("/api/auth") + handler.GET("/google/login", server.HandlerErrorFunc(h.SignIn)) + //handler.GET("/google/signup", server.HandlerErrorFunc(h.SignUp)) + handler.GET("/google/callback", server.HandlerErrorFunc(h.Callback)) + //handler.GET("/google/invite", server.HandlerErrorFunc(h.JoinToTenant)) + //adminHandler := route.Group("/api/auth") + //adminHandler.Use(authMiddleware) + //adminHandler.POST("/google/invite", server.HandlerErrorFuncAuth(h.Invite)) +} + +// SignIn login using google auth +// @Summary login using google auth +// @Description login using google auth +// @Tags Auth +// @ID auth_login +// @Param redirect_url query string false "redirect base url" +// @Produce json +// @Success 200 {object} string +// @Router /auth/google/login [get] +func (h *AuthHandler) SignIn(c *gin.Context) error { + var param parameters.LoginParam + if err := c.ShouldBindQuery(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong redirect url") + } + buf, err := json.Marshal(parameters.OAuthParam{Action: oauth.LoginState}) + if err != nil { + return utils.Internal.Wrap(err, "can not marshal payload") + } + state := base64.URLEncoding.EncodeToString(buf) + url, err := h.oauthClient.GetAuthURL(c.Request.Context(), param.RedirectURL, state) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, url) +} + +func (h *AuthHandler) RefreshToken(c *gin.Context) error { + //buf, err := json.Marshal(parameters.OAuthParam{Action: oauth.LoginState}) + //if err != nil { + // return utils.Internal.Wrap(err, "can not marshal payload") + //} + //state := base64.URLEncoding.EncodeToString(buf) + //url, err := h.oauthClient.RefreshToken(c.Request.Context(), state) + //if err != nil { + // return err + //} + //c.Redirect(http.StatusFound, url) + return nil +} + +func (h *AuthHandler) Callback(c *gin.Context) error { + code := c.Query(oauth.CodeNameGoogle) + + buf, err := base64.URLEncoding.DecodeString(c.Query(oauth.StateNameGoogle)) + if err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong state") + } + var state parameters.OAuthParam + if err = json.Unmarshal(buf, &state); err != nil { + return utils.Internal.Wrap(err, "can not unmarshal OAuth state") + } + + response, err := h.oauthClient.ExchangeCode(c.Request.Context(), code) + if err != nil { + return err + } + var user *model.User + switch state.Action { + case oauth.LoginState: + user, err = h.authBL.QuickLogin(c.Request.Context(), response) + case oauth.SignUpState: + user, err = h.authBL.SignUp(c.Request.Context(), response) + //case oauth.InviteState: + // user, err = h.authBL.JoinToTenant(c.Request.Context(), &state, response) + default: + err = fmt.Errorf("unknown state %s ", state.Action) + } + if err != nil { + return err + } + identity := &security.Identity{ + AccessToken: response.AccessToken, + RefreshToken: response.RefreshToken, + User: &model.User{ID: user.ID, + TenantID: user.TenantID, + }, + } + token, err := h.jwtService.Create(identity) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, token) +} + +//SignUp register new user and tenant using google auth +//@Summary register new user and tenant using google auth +//@Description register new user and tenant using google auth +//@Tags Auth +//@ID auth_signup +//@Produce json +//@Success 200 {object} string +//@Router /auth/google/signup [get] +//func (h *AuthHandler) SignUp(c *gin.Context) error { +// buf, err := json.Marshal(parameters.OAuthParam{Action: oauth.SignUpState}) +// if err != nil { +// return utils.Internal.Wrap(err, "can not marshal payload") +// } +// +// state := base64.URLEncoding.EncodeToString(buf) +// url, err := h.oauthClient.GetAuthURL(c.Request.Context(), state) +// if err != nil { +// return err +// } +// c.Redirect(http.StatusFound, url) +// return nil +//} + +// Invite create invitation for user +// @Summary create invitation for user +// @Description create invitation for user +// @Tags Auth +// @ID auth_invitation +// @Param params body parameters.InviteParam true "invitation parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} string +// @Router /auth/google/invite [post] +//func (h *AuthHandler) Invite(c *gin.Context, identity *security.Identity) error { +// var param parameters.InviteParam +// if err := c.BindJSON(¶m); err != nil { +// return utils.ErrorBadRequest.Wrap(err, "can not parse payload") +// } +// if err := param.Validate(); err != nil { +// return utils.ErrorBadRequest.Wrap(err, err.Error()) +// } +// +// url, err := h.authBL.Invite(c.Request.Context(), identity, ¶m) +// if err != nil { +// return err +// } +// return server.JsonResult(c, http.StatusOK, url) +//} + +// JoinToTenant join user to tenant using invitation link +// @Summary join user to tenant using invitation link +// @Description join user to tenant using invitation link +// @Tags Auth +// @ID auth_join_to_team +// @Produce json +// @Success 200 {object} string +// @Router /auth/google/invite [get] +//func (h *AuthHandler) JoinToTenant(c *gin.Context) error { +// //param := c.Query("state") +// // +// //key, err := base64.URLEncoding.DecodeString(param) +// //if err != nil { +// // return utils.ErrorBadRequest.Wrap(err, "wrong state") +// //} +// ////value, err := h.storage.Pull(string(key)) +// // +// //state := base64.URLEncoding.EncodeToString("value") +// // +// //url, err := h.oauthClient.GetAuthURL(c.Request.Context(), state) +// //if err != nil { +// // return err +// //} +// //c.Redirect(http.StatusFound, url) +// return nil +//} diff --git a/src/backend/api/handler/chat.go b/src/backend/api/handler/chat.go new file mode 100644 index 00000000..bc86369b --- /dev/null +++ b/src/backend/api/handler/chat.go @@ -0,0 +1,148 @@ +package handler + +import ( + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/gin-gonic/gin" + "io" + "net/http" + "strconv" +) + +type ChatHandler struct { + chatBL bll.ChatBL +} + +func NewChatHandler(chatBL bll.ChatBL) *ChatHandler { + return &ChatHandler{chatBL: chatBL} + +} + +func (h *ChatHandler) Mount(route *gin.Engine, authMiddleware gin.HandlerFunc) { + handler := route.Group("/api/chats") + handler.Use(authMiddleware) + handler.GET("/get-user-chat-sessions", server.HandlerErrorFuncAuth(h.GetSessions)) + handler.GET("/get-chat-session/:id", server.HandlerErrorFuncAuth(h.GetByID)) + handler.POST("/create-chat-session", server.HandlerErrorFuncAuth(h.CreateSession)) + handler.POST("/send-message", server.HandlerErrorFuncAuth(h.SendMessage)) + handler.POST("/message/feedback", server.HandlerErrorFuncAuth(h.MessageFeedback)) +} + +// GetSessions return list of chat sessions for current user +// @Summary return list of chat sessions for current user +// @Description return list of chat sessions for current user +// @Tags Chat +// @ID chat_get_sessions +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {array} model.ChatSession +// @Router /chats/get-user-chat-sessions [get] +func (h *ChatHandler) GetSessions(c *gin.Context, identity *security.Identity) error { + sessions, err := h.chatBL.GetSessions(c.Request.Context(), identity.User) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, sessions) +} + +// GetByID return chat session with messages by given id +// @Summary return chat session with messages by given id +// @Description return chat session with messages by given id +// @Tags Chat +// @ID chat_get_by_id +// @Param id path int true "session id" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.ChatSession +// @Router /chats/get-chat-session/{id} [get] +func (h *ChatHandler) GetByID(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + session, err := h.chatBL.GetSessionByID(c.Request.Context(), identity.User, id) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, session) +} + +// CreateSession creates new chat session +// @Summary creates new chat session +// @Description creates new chat session +// @Tags Chat +// @ID chat_create_session +// @Param payload body parameters.CreateChatSession true "create session parameters" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.ChatSession +// @Router /chats/create-chat-session [post] +func (h *ChatHandler) CreateSession(c *gin.Context, identity *security.Identity) error { + var param parameters.CreateChatSession + if err := server.BindJsonAndValidate(c, ¶m); err != nil { + return err + } + session, err := h.chatBL.CreateSession(c.Request.Context(), identity.User, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusCreated, session) +} + +// SendMessage send message and wait stream response +// @Summary send message and wait stream response +// @Description send message and wait stream response +// @Tags Chat +// @ID chat_send_message +// @Param payload body parameters.CreateChatMessageRequest true "send message parameters" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.ChatMessage +// @Router /chats/send-message [post] +func (h *ChatHandler) SendMessage(c *gin.Context, identity *security.Identity) error { + var param parameters.CreateChatMessageRequest + if err := server.BindJsonAndValidate(c, ¶m); err != nil { + return err + } + assistant, err := h.chatBL.SendMessage(c, identity.User, ¶m) + if err != nil { + return err + } + c.Stream(func(w io.Writer) bool { + response, ok := assistant.Receive() + if !ok { + return false + } + c.SSEvent(response.Type, response) + return ok + }) + return nil +} + +// MessageFeedback add feedback to message +// @Summary add feedback to message +// @Description add feedback to message +// @Tags Chat +// @ID chat_message_feedback +// @Param payload body parameters.MessageFeedbackParam true "send message parameters" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.ChatMessageFeedback +// @Router /chats/message/feedback [post] +func (h *ChatHandler) MessageFeedback(c *gin.Context, identity *security.Identity) error { + var param parameters.MessageFeedbackParam + if err := c.BindJSON(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong payload") + } + if err := param.Validate(); err != nil { + return utils.ErrorBadRequest.Wrap(err, err.Error()) + } + feedback, err := h.chatBL.FeedbackMessage(c, identity.User, param.ID.IntPart(), param.Vote == parameters.MessageFeedbackUpvote) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, feedback) +} diff --git a/src/backend/api/handler/connector.go b/src/backend/api/handler/connector.go new file mode 100644 index 00000000..df1c72cb --- /dev/null +++ b/src/backend/api/handler/connector.go @@ -0,0 +1,168 @@ +package handler + +import ( + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/gin-gonic/gin" + "net/http" + "strconv" +) + +type ConnectorHandler struct { + connectorBL bll.ConnectorBL +} + +func NewCollectorHandler(connectorBL bll.ConnectorBL) *ConnectorHandler { + return &ConnectorHandler{ + connectorBL: connectorBL, + } +} +func (h *ConnectorHandler) Mount(route *gin.Engine, authMiddleware gin.HandlerFunc) { + handler := route.Group("/api/manage/connector") + handler.Use(authMiddleware) + handler.GET("/source_types", server.HandlerErrorFuncAuth(h.GetSourceTypes)) + handler.GET("/", server.HandlerErrorFuncAuth(h.GetAll)) + handler.GET("/:id", server.HandlerErrorFuncAuth(h.GetById)) + handler.POST("/", server.HandlerErrorFuncAuth(h.Create)) + handler.PUT("/:id", server.HandlerErrorFuncAuth(h.Update)) + handler.POST("/:id/:action", server.HandlerErrorFuncAuth(h.Archive)) +} + +// GetAll return list of allowed connectors +// @Summary return list of allowed connectors +// @Description return list of allowed connectors +// @Tags Connectors +// @ID connectors_get_all +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {array} model.Connector +// @Router /manage/connector [get] +func (h *ConnectorHandler) GetAll(c *gin.Context, identity *security.Identity) error { + connectors, err := h.connectorBL.GetAll(c.Request.Context(), identity.User) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, connectors) +} + +// GetById return list of allowed connectors +// @Summary return list of allowed connectors +// @Description return list of allowed connectors +// @Tags Connectors +// @ID connectors_get_by_id +// @Produce json +// @Param id path int true "connector id" +// @Security ApiKeyAuth +// @Success 200 {object} model.Connector +// @Router /manage/connector/{id} [get] +func (h *ConnectorHandler) GetById(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + + connectors, err := h.connectorBL.GetByID(c.Request.Context(), identity.User, id) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, connectors) +} + +// Create creates connector +// @Summary creates connector +// @Description creates connector +// @Tags Connectors +// @ID connectors_create +// @Param params body parameters.CreateConnectorParam true "connector create parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 201 {object} model.Connector +// @Router /manage/connector/ [post] +func (h *ConnectorHandler) Create(c *gin.Context, identity *security.Identity) error { + var param parameters.CreateConnectorParam + if err := c.BindJSON(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong payload") + } + if err := param.Validate(); err != nil { + return utils.ErrorBadRequest.Wrap(err, err.Error()) + } + connector, err := h.connectorBL.Create(c.Request.Context(), identity.User, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusCreated, connector) +} + +// Update updates connector +// @Summary updates connector +// @Description updates connector +// @Tags Connectors +// @ID connectors_update +// @Param id path int true "connector id" +// @Param params body parameters.UpdateConnectorParam true "connector update parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.Connector +// @Router /manage/connector/{id} [put] +func (h *ConnectorHandler) Update(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + var param parameters.UpdateConnectorParam + if err = c.BindJSON(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong payload") + } + if err = param.Validate(); err != nil { + return utils.ErrorBadRequest.Wrapf(err, "validation error %s", err.Error()) + } + connector, err := h.connectorBL.Update(c.Request.Context(), id, identity.User, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, connector) +} + +// GetSourceTypes return list of source types +// @Summary return list of source types +// @Description return list of source types +// @Tags Connectors +// @ID connectors_get_source_types +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {array} model.SourceTypeDescription +// @Router /manage/connector/source_types [get] +func (h *ConnectorHandler) GetSourceTypes(c *gin.Context, identity *security.Identity) error { + return server.JsonResult(c, http.StatusOK, model.SourceTypesList) +} + +// Archive delete or restore connector +// @Summary delete or restore connector +// @Description delete or restore connector +// @Tags Connectors +// @ID Connectors_delete_restore +// @Param id path int true "Connectors id" +// @Param action path string true "action : delete | restore " +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.Connector +// @Router /manage/connector/{id}/{action} [post] +func (h *ConnectorHandler) Archive(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + action := c.Param("action") + if !(action == ActionRestore || action == ActionDelete) { + return utils.ErrorBadRequest.Newf("invalid action: should be %s or %s", ActionRestore, ActionDelete) + } + connector, err := h.connectorBL.Archive(c.Request.Context(), identity.User, id, action == ActionRestore) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, connector) +} diff --git a/src/backend/api/handler/document.go b/src/backend/api/handler/document.go new file mode 100644 index 00000000..482c9a04 --- /dev/null +++ b/src/backend/api/handler/document.go @@ -0,0 +1,72 @@ +package handler + +import ( + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "fmt" + "github.com/gin-gonic/gin" + "mime/multipart" + "net/http" + "sync" +) + +type DocumentHandler struct { + documentBL bll.DocumentBL +} + +func NewDocumentHandler(documentBL bll.DocumentBL) *DocumentHandler { + return &DocumentHandler{documentBL: documentBL} +} + +func (h *DocumentHandler) Mount(router *gin.Engine, authMiddleware gin.HandlerFunc) { + handler := router.Group("/api/manage/documents").Use(authMiddleware) + handler.GET("/", server.HandlerErrorFuncAuth(h.GetAll)) + handler.POST("/upload", server.HandlerErrorFuncAuth(h.Upload)) +} + +func (h *DocumentHandler) GetAll(c *gin.Context, identity *security.Identity) error { + return nil +} + +func (h *DocumentHandler) Upload(c *gin.Context, identity *security.Identity) error { + form, _ := c.MultipartForm() + files := form.File["upload[]"] + + wg := sync.WaitGroup{} + result := make([]*parameters.DocumentUploadResponse, len(files)) + for i, f := range files { + wg.Add(1) + go func(idx int, file multipart.FileHeader) { + defer wg.Done() + fileReader, err := file.Open() + contentType := file.Header.Get("Content-Type") + if err != nil { + result[idx] = ¶meters.DocumentUploadResponse{ + FileName: file.Filename, + Error: fmt.Sprintf("open file : %s", err.Error()), + Document: nil, + } + return + } + defer fileReader.Close() + document, err := h.documentBL.UploadDocument(c.Request.Context(), identity.User, file.Filename, contentType, fileReader) + if err != nil { + result[idx] = ¶meters.DocumentUploadResponse{ + FileName: file.Filename, + Error: fmt.Sprintf("upload document : %s", err.Error()), + Document: nil, + } + return + } + result[idx] = ¶meters.DocumentUploadResponse{ + FileName: file.Filename, + Error: "", + Document: document, + } + }(i, *f) + } + wg.Wait() + return server.JsonResult(c, http.StatusOK, result) +} diff --git a/src/backend/api/handler/embedding_model.go b/src/backend/api/handler/embedding_model.go new file mode 100644 index 00000000..e2650f8f --- /dev/null +++ b/src/backend/api/handler/embedding_model.go @@ -0,0 +1,157 @@ +package handler + +import ( + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/gin-gonic/gin" + "net/http" + "strconv" +) + +type EmbeddingModelHandler struct { + embeddingModelBL bll.EmbeddingModelBL +} + +func NewEmbeddingModelHandler(embeddingModelBL bll.EmbeddingModelBL) *EmbeddingModelHandler { + return &EmbeddingModelHandler{embeddingModelBL: embeddingModelBL} +} + +func (h *EmbeddingModelHandler) Mount(router *gin.Engine, authMiddleware gin.HandlerFunc) { + handler := router.Group("/api/manage/embedding_models").Use(authMiddleware) + handler.GET("/", server.HandlerErrorFuncAuth(h.GetAll)) + handler.GET("/:id", server.HandlerErrorFuncAuth(h.GetByID)) + handler.POST("/", server.HandlerErrorFuncAuth(h.Create)) + handler.PUT("/:id", server.HandlerErrorFuncAuth(h.Update)) + handler.POST("/:id/:action", server.HandlerErrorFuncAuth(h.Delete)) + +} + +// GetAll return list of embedding models +// @Summary return list of embedding models +// @Description return list of embedding models +// @Tags EmbeddingModel +// @ID embedding_model_get_all +// @Param archived query bool false "true for include deleted embedding models" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {array} model.EmbeddingModel +// @Router /manage/embedding_models [get] +func (h *EmbeddingModelHandler) GetAll(c *gin.Context, identity *security.Identity) error { + var param parameters.ArchivedParam + if err := c.ShouldBindQuery(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong parameters") + } + result, err := h.embeddingModelBL.GetAll(c.Request.Context(), identity.User, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, result) +} + +// GetByID return embedding model by id +// @Summary return embedding model by id +// @Description return embedding model by id +// @Tags EmbeddingModel +// @ID embedding_model_get_by_id +// @Param id path int true "embedding model id" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.EmbeddingModel +// @Router /manage/embedding_models/{id} [get] +func (h *EmbeddingModelHandler) GetByID(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + embeddingModel, err := h.embeddingModelBL.GetByID(c.Request.Context(), identity.User, id) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, embeddingModel) +} + +// Create creates embedding models +// @Summary creates embedding models +// @Description creates embedding models +// @Tags EmbeddingModel +// @ID embedding_model_create +// @Param params body parameters.EmbeddingModelParam true "embedding model parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 201 {object} model.EmbeddingModel +// @Router /manage/embedding_models [post] +func (h *EmbeddingModelHandler) Create(c *gin.Context, identity *security.Identity) error { + var param parameters.EmbeddingModelParam + if err := c.ShouldBind(¶m); err != nil { + return utils.ErrorBadRequest.New("invalid params") + } + embeddingModel, err := h.embeddingModelBL.Create(c.Request.Context(), identity.User, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusCreated, embeddingModel) +} + +// Update updates embedding model +// @Summary updates embedding model +// @Description updates embedding model +// @Tags EmbeddingModel +// @ID embedding_model_update +// @Param id path int true "embedding model id" +// @Param params body parameters.EmbeddingModelParam true "embedding model parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.EmbeddingModel +// @Router /manage/embedding_models/{id} [put] +func (h *EmbeddingModelHandler) Update(c *gin.Context, identity *security.Identity) error { + var param parameters.EmbeddingModelParam + if err := c.ShouldBind(¶m); err != nil { + return utils.ErrorBadRequest.New("invalid params") + } + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + embeddingModel, err := h.embeddingModelBL.Update(c.Request.Context(), identity.User, id, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, embeddingModel) +} + +// Delete deletes or restore embedding model +// @Summary delete or restore embedding model +// @Description delete or restore embedding model +// @Tags EmbeddingModel +// @ID embedding_model_delete +// @Param id path int true "embedding model id" +// @Param action path string true "action : delete | restore " +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.EmbeddingModel +// @Router /manage/embedding_models/{id}/{action} [post] +func (h *EmbeddingModelHandler) Delete(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + action := c.Param("action") + if !(action == ActionRestore || action == ActionDelete) { + return utils.ErrorBadRequest.Newf("invalid action: should be %s or %s", ActionRestore, ActionDelete) + } + var embedingModel *model.EmbeddingModel + switch action { + case ActionRestore: + embedingModel, err = h.embeddingModelBL.Restore(c.Request.Context(), identity.User, id) + case ActionDelete: + embedingModel, err = h.embeddingModelBL.Delete(c.Request.Context(), identity.User, id) + } + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, embedingModel) +} diff --git a/src/backend/api/handler/model.go b/src/backend/api/handler/model.go new file mode 100644 index 00000000..5beb7e08 --- /dev/null +++ b/src/backend/api/handler/model.go @@ -0,0 +1,10 @@ +package handler + +const ( + ActionDelete = "delete" + ActionRestore = "restore" +) + +type TokenResponse struct { + Token string `json:"token"` +} diff --git a/src/backend/api/handler/oauth.go b/src/backend/api/handler/oauth.go new file mode 100644 index 00000000..680432e4 --- /dev/null +++ b/src/backend/api/handler/oauth.go @@ -0,0 +1,85 @@ +package handler + +import ( + "cognix.ch/api/v2/core/oauth" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/gin-gonic/gin" + "golang.org/x/oauth2" + "net/http" +) + +// OAuthHandler provide oauth authentication for third part services +type OAuthHandler struct { + oauthConfig *oauth.Config +} + +func NewOAuthHandler(oauthConfig *oauth.Config) *OAuthHandler { + return &OAuthHandler{ + oauthConfig: oauthConfig, + } +} + +func (h *OAuthHandler) Mount(route *gin.Engine) { + handler := route.Group("/api/oauth") + handler.GET("/:provider/auth_url", server.HandlerErrorFunc(h.GetUrl)) + //handler.GET("/google/signup", server.HandlerErrorFunc(h.SignUp)) + handler.GET("/:provider/callback", server.HandlerErrorFunc(h.Callback)) + handler.POST("/:provider/refresh_token", server.HandlerErrorFunc(h.Refresh)) +} + +func (h *OAuthHandler) GetUrl(c *gin.Context) error { + provider := c.Param("provider") + var param parameters.LoginParam + if err := c.ShouldBindQuery(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong redirect url") + } + + oauthClient, err := oauth.NewProvider(provider, h.oauthConfig) + if err != nil { + return utils.Internal.Wrap(err, "unknown provider") + } + url, err := oauthClient.GetAuthURL(c.Request.Context(), param.RedirectURL, "") + if err != nil { + return err + } + return server.StringResult(c, http.StatusOK, []byte(url)) +} + +func (h *OAuthHandler) Callback(c *gin.Context) error { + provider := c.Param("provider") + query := make(map[string]string) + if err := c.BindQuery(&query); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong payload") + } + + oauthClient, err := oauth.NewProvider(provider, h.oauthConfig) + if err != nil { + return utils.Internal.Wrap(err, "unknown provider") + } + result, err := oauthClient.ExchangeCode(c.Request.Context(), query["code"]) + if err != nil { + return utils.ErrorPermission.New(err.Error()) + } + return server.JsonResult(c, http.StatusOK, result) +} + +func (h *OAuthHandler) Refresh(c *gin.Context) error { + provider := c.Param("provider") + var token oauth2.Token + if err := c.BindJSON(&token); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong payload") + } + oauthClient, err := oauth.NewProvider(provider, h.oauthConfig) + if err != nil { + return utils.Internal.Wrap(err, "unknown provider") + } + + result, err := oauthClient.RefreshToken(&token) + if err != nil { + return utils.ErrorPermission.New(err.Error()) + } + _ = provider + return server.JsonResult(c, http.StatusOK, result) +} diff --git a/src/backend/api/handler/persona.go b/src/backend/api/handler/persona.go new file mode 100644 index 00000000..2986ed90 --- /dev/null +++ b/src/backend/api/handler/persona.go @@ -0,0 +1,158 @@ +package handler + +import ( + "cognix.ch/api/v2/core/ai" + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/gin-gonic/gin" + "net/http" + "strconv" +) + +type PersonaHandler struct { + personaBL bll.PersonaBL + aiBuilder *ai.Builder +} + +func NewPersonaHandler(personaBL bll.PersonaBL, aiBuilder *ai.Builder) *PersonaHandler { + return &PersonaHandler{personaBL: personaBL, aiBuilder: aiBuilder} +} +func (h *PersonaHandler) Mount(route *gin.Engine, authMiddleware gin.HandlerFunc) { + handler := route.Group("/api/manage/personas") + handler.Use(authMiddleware) + handler.GET("/", server.HandlerErrorFuncAuth(h.GetAll)) + handler.GET("/:id", server.HandlerErrorFuncAuth(h.GetByID)) + handler.POST("/", server.HandlerErrorFuncAuth(h.Create)) + handler.PUT("/:id", server.HandlerErrorFuncAuth(h.Update)) + handler.POST("/:id/:action", server.HandlerErrorFuncAuth(h.Archive)) +} + +// GetAll return list of allowed personas +// @Summary return list of allowed personas +// @Description return list of allowed personas +// @Tags Persona +// @ID personas_get_all +// @param archived query bool false "true for include deleted personas" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {array} model.Persona +// @Router /manage/personas [get] +func (h *PersonaHandler) GetAll(c *gin.Context, identity *security.Identity) error { + var param parameters.ArchivedParam + if err := c.BindQuery(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong parameters") + } + personas, err := h.personaBL.GetAll(c.Request.Context(), identity.User, param.Archived) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, personas) + +} + +// GetByID return persona by id +// @Summary return persona by id +// @Description return persona by id +// @Tags Persona +// @ID persona_get_by_id +// @Param id path int true "persona id" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.Persona +// @Router /manage/personas/{id} [get] +func (h *PersonaHandler) GetByID(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + persona, err := h.personaBL.GetByID(c.Request.Context(), identity.User, id) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, persona) +} + +// Create create persona +// @Summary create persona +// @Description create persona +// @Tags Persona +// @ID persona_create +// @Param id body parameters.PersonaParam true "persona payload" +// @Produce json +// @Security ApiKeyAuth +// @Success 201 {object} model.Persona +// @Router /manage/personas [post] +func (h *PersonaHandler) Create(c *gin.Context, identity *security.Identity) error { + var param parameters.PersonaParam + if err := server.BindJsonAndValidate(c, ¶m); err != nil { + return err + } + persona, err := h.personaBL.Create(c.Request.Context(), identity.User, ¶m) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusCreated, persona) +} + +// Update update persona +// @Summary update persona +// @Description update persona +// @Tags Persona +// @ID persona_update +// @Param id path int true "persona id" +// @Param id body parameters.PersonaParam true "persona payload" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.Persona +// @Router /manage/personas/{id} [put] +func (h *PersonaHandler) Update(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + var param parameters.PersonaParam + if err = server.BindJsonAndValidate(c, ¶m); err != nil { + return err + } + persona, err := h.personaBL.Update(c.Request.Context(), id, identity.User, ¶m) + if err != nil { + return err + } + if persona.LLM != nil { + h.aiBuilder.Invalidate(persona.LLM) + } + return server.JsonResult(c, http.StatusOK, persona) +} + +// Archive delete or restore persona +// @Summary delete or restore persona +// @Description delete or restore persona +// @Tags Persona +// @ID persona_delete_restore +// @Param id path int true "persona id" +// @Param action path string true "action : delete | restore " +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.Persona +// @Router /manage/personas/{id}/{action} [post] +func (h *PersonaHandler) Archive(c *gin.Context, identity *security.Identity) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id == 0 { + return utils.ErrorBadRequest.New("id should be presented") + } + action := c.Param("action") + if !(action == ActionRestore || action == ActionDelete) { + return utils.ErrorBadRequest.Newf("invalid action: should be %s or %s", ActionRestore, ActionDelete) + } + persona, err := h.personaBL.Archive(c.Request.Context(), identity.User, id, action == ActionRestore) + if err != nil { + return err + } + if persona.LLM != nil { + h.aiBuilder.Invalidate(persona.LLM) + } + return server.JsonResult(c, http.StatusOK, persona) +} diff --git a/src/backend/api/handler/router.go b/src/backend/api/handler/router.go new file mode 100644 index 00000000..abeebd16 --- /dev/null +++ b/src/backend/api/handler/router.go @@ -0,0 +1 @@ +package handler diff --git a/src/backend/api/handler/swagger.go b/src/backend/api/handler/swagger.go new file mode 100644 index 00000000..4551d453 --- /dev/null +++ b/src/backend/api/handler/swagger.go @@ -0,0 +1,49 @@ +package handler + +import ( + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "encoding/json" + "net/http" + + _ "cognix.ch/api/v2/api/docs" + + "github.com/gin-gonic/gin" + swaggerfiles "github.com/swaggo/files" + ginSwagger "github.com/swaggo/gin-swagger" + "github.com/swaggo/swag" +) + +type SwaggerHandler struct { +} + +func NewSwaggerHandler() *SwaggerHandler { + return &SwaggerHandler{} +} + +func (h *SwaggerHandler) Mount(router *gin.Engine) { + url := ginSwagger.URL("docs/doc.json") + ///url := ginSwagger.URL("http://localhost:8080/swagger/doc.json") + router.GET("/api/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler, url)) + router.GET("/swagger/docs", server.HandlerErrorFunc(h.GetDoc)) + router.GET("/api/health", h.Health) + +} + +// GetDoc returns the swagger doc in the json format. +func (h *SwaggerHandler) GetDoc(c *gin.Context) error { + jsonDoc, err := swag.ReadDoc() + if err != nil { + return utils.NotFound.Wrap(err, "swagger doc not found") + } + var result map[string]interface{} + err = json.Unmarshal([]byte(jsonDoc), &result) + if err != nil { + return utils.NotFound.Wrap(err, "can not parse swagger doc") + } + return server.JsonResult(c, http.StatusOK, result) +} + +func (h *SwaggerHandler) Health(c *gin.Context) { + c.String(http.StatusOK, "ok") +} diff --git a/src/backend/api/handler/tenant.go b/src/backend/api/handler/tenant.go new file mode 100644 index 00000000..a2f474f0 --- /dev/null +++ b/src/backend/api/handler/tenant.go @@ -0,0 +1,126 @@ +package handler + +import ( + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "net/http" +) + +// TenantHandler handles authentication endpoints +type TenantHandler struct { + tenantBL bll.TenantBL +} + +func NewTenantHandler(TenantBL bll.TenantBL) *TenantHandler { + return &TenantHandler{ + tenantBL: TenantBL, + } +} + +func (h *TenantHandler) Mount(route *gin.Engine, TenantMiddleware gin.HandlerFunc) { + handler := route.Group("/api/tenant") + handler.Use(TenantMiddleware) + handler.GET("/users", server.HandlerErrorFuncAuth(h.GetUserList)) + handler.GET("/user_info", server.HandlerErrorFuncAuth(h.GetUserInfo)) + handler.POST("/users", server.HandlerErrorFuncAuth(h.AddUser)) + handler.PUT("/users/:id", server.HandlerErrorFuncAuth(h.EditUser)) +} + +// GetUserList return list of users +// @Summary return list of users +// @Description return list of users +// @Tags Tenant +// @ID tenant_get_users +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {array} model.User +// @Router /tenant/users [get] +func (h *TenantHandler) GetUserList(c *gin.Context, identity *security.Identity) error { + users, err := h.tenantBL.GetUsers(c.Request.Context(), identity.User) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, users) +} + +// AddUser add new user +// @Summary add new user +// @Description add new user +// @Tags Tenant +// @ID tenant_add_user +// @Param params body parameters.AddUserParam true "create user parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.User +// @Router /tenant/users [post] +func (h *TenantHandler) AddUser(c *gin.Context, identity *security.Identity) error { + if !identity.User.HasRoles(model.RoleAdmin, model.RoleSuperAdmin) { + return utils.ErrorPermission.New("do not have permission") + } + + var param parameters.AddUserParam + if err := c.ShouldBind(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong parameter") + } + if err := param.Validate(); err != nil { + return utils.ErrorBadRequest.New(err.Error()) + } + user, err := h.tenantBL.AddUser(c.Request.Context(), identity.User, param.Email, param.Role) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, user) +} + +// EditUser edit user +// @Summary edit user +// @Description edit user +// @Tags Tenant +// @ID tenant_edit_user +// @Param id path string true "user id" +// @Param params body parameters.EditUserParam true "edit user parameter" +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.User +// @Router /tenant/users/{id} [put] +func (h *TenantHandler) EditUser(c *gin.Context, identity *security.Identity) error { + if !identity.User.HasRoles(model.RoleAdmin, model.RoleSuperAdmin) { + return utils.ErrorPermission.New("do not have permission") + } + id := c.Param("id") + userID, err := uuid.Parse(id) + if err != nil { + return utils.ErrorPermission.Wrap(err, "wrong id parameter") + } + var param parameters.EditUserParam + if err := c.ShouldBind(¶m); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong parameter") + } + if err := param.Validate(); err != nil { + return utils.ErrorBadRequest.New(err.Error()) + } + user, err := h.tenantBL.UpdateUser(c.Request.Context(), identity.User, userID, param.Role) + if err != nil { + return err + } + return server.JsonResult(c, http.StatusOK, user) +} + +// GetUserInfo get user info +// @Summary get user info +// @Description get user info +// @Tags Tenant +// @ID tenant_get_user_info +// @Produce json +// @Security ApiKeyAuth +// @Success 200 {object} model.User +// @Router /tenant/user_info [get] +func (h *TenantHandler) GetUserInfo(c *gin.Context, identity *security.Identity) error { + return server.JsonResult(c, http.StatusOK, identity.User) +} diff --git a/src/backend/api/main.go b/src/backend/api/main.go new file mode 100644 index 00000000..16a4d3bf --- /dev/null +++ b/src/backend/api/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" + _ "go.uber.org/fx" +) + +// @title Cognix API +// @version 1.0 +// @description This is Cognix Golang API Documentation + +// @contact.name API Support +// @contact.url +// @contact.email + +// @securityDefinitions.apikey ApiKeyAuth +// @in header +// @name Authorization + +// @BasePath /api +// @query.collection.format multi + +func main() { + utils.InitLogger(true) + app := fx.New(Module) // fx.WithLogger(func(log *zap.Logger) fxevent.Logger { + // return &fxevent.ZapLogger{Logger: log} + //}) + + app.Run() +} diff --git a/src/backend/api/model.go b/src/backend/api/model.go new file mode 100644 index 00000000..5e6012aa --- /dev/null +++ b/src/backend/api/model.go @@ -0,0 +1,55 @@ +package main + +import ( + "cognix.ch/api/v2/api/handler" + "cognix.ch/api/v2/core/oauth" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/utils" + "github.com/caarlos0/env/v10" + "github.com/gin-gonic/gin" + "go.uber.org/fx" +) + +type MountParams struct { + fx.In + Router *gin.Engine + AuthMiddleware *server.AuthMiddleware + AutHandler *handler.AuthHandler + SwaggerHandler *handler.SwaggerHandler + ConnectorHandler *handler.ConnectorHandler + ChatHandler *handler.ChatHandler + PersonaHandler *handler.PersonaHandler + EmbeddingModelHandler *handler.EmbeddingModelHandler + TenantHandler *handler.TenantHandler + DocumentHandler *handler.DocumentHandler + OAuthHandler *handler.OAuthHandler +} + +type Config struct { + DB *repository.Config + OAuth *oauth.Config + Debug bool `env:"DEBUG" envDefault:"false"` + Port int `env:"PORT" envDefault:"8080"` + StoragePath string `env:"STORAGE_PATH"` + RedirectURL string `env:"REDIRECT_URL"` + JWTSecret string `env:"JWT_SECRET" envDefault:"secret"` + JWTExpiredTime int `env:"JWT_EXPIRED_TIME" envDefault:"60"` +} + +func ReadConfig() (*Config, error) { + cfg := &Config{ + DB: &repository.Config{}, + OAuth: &oauth.Config{ + Microsoft: &oauth.MicrosoftConfig{}, + Google: &oauth.GoogleConfig{}, + }, + } + err := env.Parse(cfg) + if err != nil { + return nil, err + } + utils.InitLogger(cfg.Debug) + return cfg, nil + +} diff --git a/src/backend/api/module.go b/src/backend/api/module.go new file mode 100644 index 00000000..ab0414bc --- /dev/null +++ b/src/backend/api/module.go @@ -0,0 +1,100 @@ +package main + +import ( + "cognix.ch/api/v2/api/handler" + "cognix.ch/api/v2/core/ai" + "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/oauth" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/server" + "cognix.ch/api/v2/core/storage" + "fmt" + "github.com/gin-contrib/cors" + "github.com/gin-gonic/gin" + "github.com/uptrace/opentelemetry-go-extra/otelzap" + "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" + "go.uber.org/fx" + "net/http" +) + +var Module = fx.Options( + repository.DatabaseModule, + repository.RepositoriesModule, + bll.BLLModule, + storage.MinioModule, + messaging.NatsModule, + ai.EmbeddingModule, + storage.MilvusModule, + fx.Provide(ReadConfig, + NewRouter, + newGoogleOauthProvider, + newJWTService, + //newStorage, + ai.NewBuilder, + server.NewAuthMiddleware, + handler.NewAuthHandler, + handler.NewCollectorHandler, + handler.NewSwaggerHandler, + handler.NewPersonaHandler, + handler.NewChatHandler, + handler.NewEmbeddingModelHandler, + handler.NewTenantHandler, + handler.NewDocumentHandler, + newOauthHandler, + ), + fx.Invoke( + MountRoute, + RunServer, + ), +) + +func MountRoute(param MountParams) error { + param.AutHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.SwaggerHandler.Mount(param.Router) + param.ConnectorHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.ChatHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.PersonaHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.EmbeddingModelHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.TenantHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.DocumentHandler.Mount(param.Router, param.AuthMiddleware.RequireAuth) + param.OAuthHandler.Mount(param.Router) + return nil +} + +func newGoogleOauthProvider(cfg *Config) oauth.Proxy { + return oauth.NewGoogleProvider(cfg.OAuth.Google, cfg.RedirectURL) +} +func newJWTService(cfg *Config) security.JWTService { + return security.NewJWTService(cfg.JWTSecret, cfg.JWTExpiredTime) +} + +// func newStorage(cfg *Config) (storage.Storage, error) { +// return storage.NewNutsDbStorage(cfg.StoragePath) +// } +func newOauthHandler(cfg *Config) *handler.OAuthHandler { + return handler.NewOAuthHandler(cfg.OAuth) +} +func NewRouter() *gin.Engine { + router := gin.Default() + router.Use(otelgin.Middleware("service-name")) + corsConfig := cors.DefaultConfig() + + corsConfig.CustomSchemas = cors.DefaultSchemas + corsConfig.AllowAllOrigins = true + corsConfig.AllowCredentials = true + corsConfig.AllowWildcard = true + router.Use(cors.New(corsConfig)) + return router +} + +func RunServer(cfg *Config, router *gin.Engine) { + srv := http.Server{} + srv.Addr = fmt.Sprintf("0.0.0.0:%d", cfg.Port) + srv.Handler = router + otelzap.S().Infof("Start server %s ", srv.Addr) + if err := srv.ListenAndServe(); err != nil { + otelzap.S().Errorf("HTTP server: %s", err.Error()) + } +} diff --git a/src/backend/api/test.json b/src/backend/api/test.json new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/connector/executor.go b/src/backend/connector/executor.go new file mode 100644 index 00000000..e95c9f75 --- /dev/null +++ b/src/backend/connector/executor.go @@ -0,0 +1,266 @@ +package main + +import ( + "bytes" + "cognix.ch/api/v2/core/connector" + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "cognix.ch/api/v2/core/utils" + "context" + "fmt" + "github.com/go-pg/pg/v10" + "github.com/go-resty/resty/v2" + proto2 "github.com/golang/protobuf/proto" + "github.com/nats-io/nats.go/jetstream" + "go.uber.org/zap" + "strings" + + "io" + "time" +) + +type Executor struct { + cfg *Config + connectorRepo repository.ConnectorRepository + docRepo repository.DocumentRepository + msgClient messaging.Client + minioClient storage.MinIOClient + milvusClient storage.MilvusClient + oauthClient *resty.Client + downloadClient *resty.Client +} + +// run this method listen messages from nats +func (e *Executor) run(streamName, topic string, task messaging.MessageHandler) { + if err := e.msgClient.Listen(context.Background(), streamName, topic, task); err != nil { + zap.S().Errorf("failed to listen[%s]: %v", topic, err) + } + return +} + +// runConnector run connector from nats message +func (e *Executor) runConnector(ctx context.Context, msg jetstream.Msg) error { + startTime := time.Now() + //ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.Header())) + var trigger proto.ConnectorRequest + + if err := proto2.Unmarshal(msg.Data(), &trigger); err != nil { + zap.S().Errorf("Error unmarshalling message: %s", err.Error()) + return err + } + // read connector model with documents, embedding model + connectorModel, err := e.connectorRepo.GetByID(ctx, trigger.GetId()) + if err != nil { + return err + } + defer func() { + zap.S().Infof("connector %s completed. elapsed time: %d ms", connectorModel.Name, time.Since(startTime)/time.Millisecond) + }() + + zap.S().Infof("receive message : %s [%d]", connectorModel.Name, connectorModel.ID.IntPart()) + // refresh token if needed + connectorModel.Status = model.ConnectorStatusWorking + + // create new instance of connector by connector model + connectorWF, err := connector.New(connectorModel, e.connectorRepo, e.cfg.OAuthURL) + if err != nil { + return err + } + if trigger.Params == nil { + trigger.Params = make(map[string]string) + } + // execute connector + resultCh := connectorWF.Execute(ctx, trigger.Params) + // read result from channel + hasSemanticMessage := false + for result := range resultCh { + var loopErr error + // empty result when channel was closed. + if result.SourceID == "" { + break + } + hasSemanticMessage = true + + // save content in minio + if result.Content != nil { + if err = e.saveContent(ctx, result); err != nil { + loopErr = err + zap.S().Errorf("failed to save content: %v", err) + } + + } + // find or create document from result + doc := e.handleResult(connectorModel, result) + // create or update document in database + if doc.ID.IntPart() != 0 { + loopErr = e.docRepo.Update(ctx, doc) + } else { + loopErr = e.docRepo.Create(ctx, doc) + } + + if loopErr != nil { + err = loopErr + zap.S().Errorf("Failed to update document: %v", loopErr) + continue + } + + // send message to chunking service + + if _, ok := model.WhisperFileTypes[result.FileType]; ok { + // send message to whisper + whisperDate := proto.WhisperData{ + Url: result.URL, + DocumentId: doc.ID.IntPart(), + ConnectorId: connectorModel.ID.IntPart(), + FileType: result.FileType, + CollectionName: connectorModel.CollectionName(), + ModelName: connectorModel.User.EmbeddingModel.ModelID, + ModelDimension: int32(connectorModel.User.EmbeddingModel.ModelDim), + } + zap.S().Infof("send message to whisper %s - %s", connectorModel.Name, result.URL) + if loopErr = e.msgClient.Publish(ctx, + e.msgClient.StreamConfig().WhisperStreamName, + e.msgClient.StreamConfig().WhisperStreamSubject, + &whisperDate); loopErr != nil { + err = loopErr + zap.S().Errorf("Failed to publish whisper : %v", loopErr) + continue + } + + } else { + // send message to semantic + semanticData := proto.SemanticData{ + Url: result.URL, + DocumentId: doc.ID.IntPart(), + ConnectorId: connectorModel.ID.IntPart(), + FileType: result.FileType, + CollectionName: connectorModel.CollectionName(), + ModelName: connectorModel.User.EmbeddingModel.ModelID, + ModelDimension: int32(connectorModel.User.EmbeddingModel.ModelDim), + } + zap.S().Infof("send message to semantic %s - %s", connectorModel.Name, result.URL) + if loopErr = e.msgClient.Publish(ctx, + e.msgClient.StreamConfig().SemanticStreamName, + e.msgClient.StreamConfig().SemanticStreamSubject, + &semanticData); loopErr != nil { + err = loopErr + zap.S().Errorf("Failed to publish semantic: %v", loopErr) + continue + } + } + } + if errr := e.deleteUnusedFiles(ctx, connectorModel); err != nil { + zap.S().Errorf("deleting unused files: %v", errr) + if err == nil { + err = errr + } + } + if err != nil { + zap.S().Errorf("failed to update documents: %v", err) + connectorModel.Status = model.ConnectorStatusUnableProcess + } else { + if !hasSemanticMessage { + connectorModel.Status = model.ConnectorStatusSuccess + } + } + connectorModel.LastUpdate = pg.NullTime{time.Now().UTC()} + + if err = e.connectorRepo.Update(ctx, connectorModel); err != nil { + return err + } + return nil +} + +func (e *Executor) deleteUnusedFiles(ctx context.Context, connector *model.Connector) error { + var ids []int64 + for _, doc := range connector.DocsMap { + if doc.IsExists || doc.ID.IntPart() == 0 { + continue + } + filepath := strings.Split(doc.URL, ":") + if len(filepath) == 3 && filepath[0] == "minio" { + if err := e.minioClient.DeleteObject(ctx, filepath[1], filepath[2]); err != nil { + return err + } + } + ids = append(ids, doc.ID.IntPart()) + } + if len(ids) > 0 { + if err := e.milvusClient.Delete(ctx, connector.CollectionName(), ids...); err != nil { + return err + } + return e.docRepo.DeleteByIDS(ctx, ids...) + } + return nil +} +func (e *Executor) saveContent(ctx context.Context, response *connector.Response) error { + + var reader io.Reader + // download file if url presented. + if response.Content.URL != "" { + fileResponse, err := e.downloadClient.R(). + SetDoNotParseResponse(true). + Get(response.Content.URL) + defer fileResponse.RawBody().Close() + if err = utils.WrapRestyError(fileResponse, err); err != nil { + return err + } + reader = fileResponse.RawBody() + } else { + // create reader from raw content + reader = bytes.NewReader(response.Content.Body) + } + + fileName, _, err := e.minioClient.Upload(ctx, response.Content.Bucket, response.Name, response.MimeType, reader) + if err != nil { + return err + } + response.URL = fmt.Sprintf("minio:%s:%s", response.Content.Bucket, fileName) + return nil +} + +func (e *Executor) handleResult(connectorModel *model.Connector, result *connector.Response) *model.Document { + doc, ok := connectorModel.DocsMap[result.SourceID] + if !ok { + doc = &model.Document{ + SourceID: result.SourceID, + ConnectorID: connectorModel.ID, + URL: result.URL, + Signature: result.Signature, + CreationDate: time.Now().UTC(), + } + connectorModel.DocsMap[result.SourceID] = doc + } else { + doc.URL = result.URL + doc.LastUpdate = pg.NullTime{time.Now().UTC()} + } + + return doc +} + +func NewExecutor( + cfg *Config, + connectorRepo repository.ConnectorRepository, + docRepo repository.DocumentRepository, + streamClient messaging.Client, + minioClient storage.MinIOClient, + milvusClient storage.MilvusClient, +) *Executor { + return &Executor{ + cfg: cfg, + connectorRepo: connectorRepo, + docRepo: docRepo, + msgClient: streamClient, + minioClient: minioClient, + milvusClient: milvusClient, + oauthClient: resty.New(). + SetTimeout(time.Minute). + SetBaseURL(cfg.OAuthURL), + downloadClient: resty.New(). + SetTimeout(time.Minute). + SetDoNotParseResponse(true), + } +} diff --git a/src/backend/connector/main.go b/src/backend/connector/main.go new file mode 100644 index 00000000..f3a41af4 --- /dev/null +++ b/src/backend/connector/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +func main() { + utils.InitLogger(true) + app := fx.New(Module) + + app.Run() +} diff --git a/src/backend/connector/module.go b/src/backend/connector/module.go new file mode 100644 index 00000000..55eb8c24 --- /dev/null +++ b/src/backend/connector/module.go @@ -0,0 +1,53 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "cognix.ch/api/v2/core/utils" + "context" + "go.uber.org/fx" + "go.uber.org/zap" +) + +type Config struct { + OAuthURL string `env:"OAUTH_URL,required"` +} + +var Module = fx.Options( + repository.DatabaseModule, + messaging.NatsModule, + storage.MinioModule, + storage.MilvusModule, + fx.Provide( + func() (*Config, error) { + cfg := Config{} + err := utils.ReadConfig(&cfg) + if err != nil { + zap.S().Errorf(err.Error()) + return nil, err + } + return &cfg, nil + }, + repository.NewConnectorRepository, + repository.NewDocumentRepository, + repository.NewEmbeddingModelRepository, + NewExecutor, + ), + fx.Invoke(RunServer), +) + +func RunServer(lc fx.Lifecycle, executor *Executor) error { + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + go executor.run(executor.msgClient.StreamConfig().ConnectorStreamName, executor.msgClient.StreamConfig().ConnectorStreamSubject, executor.runConnector) + return nil + }, + OnStop: func(ctx context.Context) error { + executor.msgClient.Close() + return nil + }, + }) + return nil +} diff --git a/src/backend/connector/readme.md b/src/backend/connector/readme.md new file mode 100644 index 00000000..1ddadb8d --- /dev/null +++ b/src/backend/connector/readme.md @@ -0,0 +1,56 @@ +we have tow services: +connector +connector orchestrator (we will change this name) +the orcherstrator cheks the connector table and decides if a connector needs to do the work for one line inside connector table +if yes it sends a NATS message so that only one connector service will take the message +Important check how to configure nats for retries and dead letter queue + + +max 3 retryes and then it shal go in error status +CONNECTOR SERVCES +shall inherit from a base connector class +it shall do the job f crarling +splitting by a fixed size with some char of the previous iteration +it will call the python service to create the embeddings +it will store the embedding in milvus +it will update the status of the connectr +we neeed to store: + embedding + original text +reference to find back the original document which shall be a link exposed to the client +milvus store with file index (i'll explain later) +we create a collection for each organization and a collection for each user +in milvus we create a collection for each tennant and one collection for each user foe the private connectors +collection name is fixed by a rule like tennant_{ID}. user_{ID} +pass opentelemetry context to nats for distributed tracing + + +### file compatibility + + - Office all + - doc, docx + - xls, xlsx + - rtf + - ppt, pptx + - pdf + - google docs all + - txt + - md + +#### optional + - ODT + - ODS + - ODP + - ODG + + +### connectors +teams chats +teams teams +sharepoint +slack +email (office/gmail) +google drive +one drive +dropbox +upload files (supported formats) diff --git a/src/backend/connector/service-deployment.yaml b/src/backend/connector/service-deployment.yaml new file mode 100644 index 00000000..9333dcd6 --- /dev/null +++ b/src/backend/connector/service-deployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: connectorservice +spec: + replicas: 1 + selector: + matchLabels: + app: connectorservice + template: + metadata: + labels: + app: connectorservice + spec: + containers: + - name: connectorservice + image: cognixacr.azurecr.io/cognix/connectorservice:main + envFrom: + - configMapRef: + name: cockroach-cli + - configMapRef: + name: minio-cli + - configMapRef: + name: milvus-cli + - configMapRef: + name: nats-cli + - configMapRef: + name: oauth-cli + - configMapRef: + name: env-configmap + imagePullPolicy: Always + volumeMounts: + - mountPath: /backend + name: storage-volume + imagePullSecrets: + - name: pullsecret + volumes: + - name: storage-volume + persistentVolumeClaim: + claimName: storage-volume +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-volume + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: azurefile-csi \ No newline at end of file diff --git a/src/backend/core/ai/builder.go b/src/backend/core/ai/builder.go new file mode 100644 index 00000000..112d2593 --- /dev/null +++ b/src/backend/core/ai/builder.go @@ -0,0 +1,31 @@ +package ai + +import ( + "cognix.ch/api/v2/core/model" + "sync" +) + +type Builder struct { + clients map[int64]OpenAIClient + mx sync.Mutex +} + +func NewBuilder() *Builder { + return &Builder{clients: make(map[int64]OpenAIClient)} +} + +func (b *Builder) New(llm *model.LLM) OpenAIClient { + b.mx.Lock() + defer b.mx.Unlock() + if client, ok := b.clients[llm.ID.IntPart()]; ok { + return client + } + client := NewOpenAIClient(llm.ModelID, llm.ApiKey) + b.clients[llm.ID.IntPart()] = client + return client +} +func (b *Builder) Invalidate(llm *model.LLM) { + b.mx.Lock() + delete(b.clients, llm.ID.IntPart()) + b.mx.Unlock() +} diff --git a/src/backend/core/ai/embedding.go b/src/backend/core/ai/embedding.go new file mode 100644 index 00000000..3fce0a3b --- /dev/null +++ b/src/backend/core/ai/embedding.go @@ -0,0 +1,12 @@ +package ai + +import ( + _ "github.com/deluan/flowllm/llms/openai" +) + +type ( + EmbeddingConfig struct { + EmbedderHost string `env:"EMBEDDER_GRPC_HOST,required"` + EmbedderPort int `env:"EMBEDDER_GRPC_PORT,required"` + } +) diff --git a/src/backend/core/ai/module.go b/src/backend/core/ai/module.go new file mode 100644 index 00000000..e640d904 --- /dev/null +++ b/src/backend/core/ai/module.go @@ -0,0 +1,31 @@ +package ai + +import ( + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/utils" + "fmt" + "go.uber.org/fx" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +var EmbeddingModule = fx.Options( + fx.Provide(func() (*EmbeddingConfig, error) { + cfg := EmbeddingConfig{} + if err := utils.ReadConfig(&cfg); err != nil { + return nil, err + } + return &cfg, nil + }, + newEmbeddingGRPCClient), +) + +func newEmbeddingGRPCClient(cfg *EmbeddingConfig) (proto.EmbedServiceClient, error) { + dialOptions := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + + conn, err := grpc.Dial(fmt.Sprintf("%s:%d", cfg.EmbedderHost, cfg.EmbedderPort), dialOptions...) + if err != nil { + return nil, err + } + return proto.NewEmbedServiceClient(conn), nil +} diff --git a/src/backend/core/ai/open-ai.go b/src/backend/core/ai/open-ai.go new file mode 100644 index 00000000..77fcb771 --- /dev/null +++ b/src/backend/core/ai/open-ai.go @@ -0,0 +1,48 @@ +package ai + +import ( + "context" + openai "github.com/sashabaranov/go-openai" +) + +type ( + Response struct { + Message string + } + OpenAIClient interface { + Request(ctx context.Context, message string) (*Response, error) + } + + openAIClient struct { + client *openai.Client + modelID string + } +) + +func (o *openAIClient) Request(ctx context.Context, message string) (*Response, error) { + + userMessage := openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleUser, + Content: message, + } + resp, err := o.client.CreateChatCompletion( + context.Background(), + openai.ChatCompletionRequest{ + Model: o.modelID, + Messages: []openai.ChatCompletionMessage{userMessage}, + }, + ) + if err != nil { + return nil, err + } + response := &Response{Message: resp.Choices[0].Message.Content} + return response, nil +} + +func NewOpenAIClient(modelID, apiKey string) OpenAIClient { + + return &openAIClient{ + client: openai.NewClient(apiKey), + modelID: modelID, + } +} diff --git a/src/backend/core/bll/auth.go b/src/backend/core/bll/auth.go new file mode 100644 index 00000000..da245f9c --- /dev/null +++ b/src/backend/core/bll/auth.go @@ -0,0 +1,156 @@ +package bll + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/oauth" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/google/uuid" + "time" +) + +type ( + AuthBL interface { + Login(ctx context.Context, userName string) (*model.User, error) + SignUp(ctx context.Context, identity *oauth.IdentityResponse) (*model.User, error) + //Invite(ctx context.Context, identity *security.Identity, param *parameters.InviteParam) (string, error) + //JoinToTenant(ctx context.Context, state *parameters.OAuthParam, response *oauth.IdentityResponse) (*model.User, error) + QuickLogin(ctx context.Context, identity *oauth.IdentityResponse) (*model.User, error) + } + authBL struct { + userRepo repository.UserRepository + cfg *Config + } +) + +func NewAuthBL(userRepo repository.UserRepository, + + cfg *Config) AuthBL { + return &authBL{ + userRepo: userRepo, + cfg: cfg, + } +} + +func (a *authBL) Login(ctx context.Context, userName string) (*model.User, error) { + user, err := a.userRepo.GetByUserName(ctx, userName) + if err != nil { + return nil, err + } + return user, nil +} + +func (a *authBL) SignUp(ctx context.Context, identity *oauth.IdentityResponse) (*model.User, error) { + exists, err := a.userRepo.IsUserExists(ctx, identity.Email) + if err != nil { + return nil, err + } + if exists { + return nil, utils.ErrorBadRequest.New("user already exists") + } + userID := uuid.New() + tenantID := uuid.New() + + // create user and default connector and embedding model + user := model.User{ + ID: userID, + TenantID: tenantID, + UserName: identity.Email, + FirstName: identity.GivenName, + LastName: identity.FamilyName, + ExternalID: identity.ID, + Roles: model.StringSlice{model.RoleSuperAdmin}, + Defaults: &model.Defaults{ + EmbeddingModel: &model.EmbeddingModel{ + TenantID: tenantID, + ModelID: a.cfg.DefaultEmbeddingModel, + ModelName: a.cfg.DefaultEmbeddingModel, + ModelDim: a.cfg.DefaultEmbeddingVectorSize, + IsActive: true, + CreationDate: time.Now().UTC(), + }, + }, + } + if user.FirstName == "" { + user.FirstName = identity.Name + } + if err = a.userRepo.RegisterUser(ctx, &user); err != nil { + return nil, err + } + + return &user, nil +} + +//func (a *authBL) Invite(ctx context.Context, identity *security.Identity, param *parameters.InviteParam) (string, error) { +// +// exists, err := a.userRepo.IsUserExists(ctx, param.Email) +// if err != nil { +// return "", err +// } +// if exists { +// return "", utils.ErrorBadRequest.New("user already registered.") +// } +// //buf, err := json.Marshal(parameters.OAuthParam{Action: oauth.InviteState, +// // Role: param.Role, +// // Email: param.Email, +// // TenantID: identity.User.TenantID.String(), +// //}) +// //if err != nil { +// // return "", utils.Internal.Wrap(err, "can not marshal payload") +// //} +// key := uuid.New() +// //if err = a.storage.Save(key.String(), buf); err != nil { +// // return "", err +// //} +// state := base64.URLEncoding.EncodeToString([]byte(key.String())) +// +// return fmt.Sprintf("%s/auth/google/invite?state=%s", a.redirectURL, state), nil +// +//} +// +//func (a *authBL) JoinToTenant(ctx context.Context, state *parameters.OAuthParam, response *oauth.IdentityResponse) (*model.User, error) { +// //if state.Email != response.Email { +// // return nil, utils.ErrorPermission.New("email is not equals to invite") +// //} +// user := model.User{ +// ID: uuid.New(), +// TenantID: uuid.MustParse(state.TenantID), +// UserName: state.Email, +// FirstName: response.GivenName, +// LastName: response.FamilyName, +// ExternalID: response.ID, +// Roles: model.StringSlice{state.Role}, +// } +// if err := a.userRepo.Create(ctx, &user); err != nil { +// return nil, err +// } +// return &user, nil +//} + +func (a *authBL) QuickLogin(ctx context.Context, identity *oauth.IdentityResponse) (*model.User, error) { + exists, err := a.userRepo.IsUserExists(ctx, identity.Email) + if err != nil { + return nil, err + } + if !exists { + return a.SignUp(ctx, identity) + } + user, err := a.userRepo.GetByUserName(ctx, identity.Email) + if err != nil { + return nil, err + } + if user.ExternalID == "" { + if identity.GivenName == "" { + user.FirstName = identity.Name + } else { + user.FirstName = identity.GivenName + } + user.LastName = identity.FamilyName + user.ExternalID = identity.ID + } + if err = a.userRepo.Update(ctx, user); err != nil { + return nil, err + } + return user, nil +} diff --git a/src/backend/core/bll/chat.go b/src/backend/core/bll/chat.go new file mode 100644 index 00000000..7107ef50 --- /dev/null +++ b/src/backend/core/bll/chat.go @@ -0,0 +1,140 @@ +package bll + +import ( + "cognix.ch/api/v2/core/ai" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/responder" + "cognix.ch/api/v2/core/storage" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "time" +) + +type ChatBL interface { + GetSessions(ctx context.Context, user *model.User) ([]*model.ChatSession, error) + GetSessionByID(ctx context.Context, user *model.User, id int64) (*model.ChatSession, error) + CreateSession(ctx context.Context, user *model.User, param *parameters.CreateChatSession) (*model.ChatSession, error) + SendMessage(ctx *gin.Context, user *model.User, param *parameters.CreateChatMessageRequest) (*responder.Manager, error) + FeedbackMessage(ctx *gin.Context, user *model.User, id int64, vote bool) (*model.ChatMessageFeedback, error) +} +type chatBL struct { + cfg *Config + chatRepo repository.ChatRepository + docRepo repository.DocumentRepository + personaRepo repository.PersonaRepository + embeddingModelRepo repository.EmbeddingModelRepository + aiBuilder *ai.Builder + embedding proto.EmbedServiceClient + milvusClinet storage.MilvusClient +} + +func (b *chatBL) FeedbackMessage(ctx *gin.Context, user *model.User, id int64, vote bool) (*model.ChatMessageFeedback, error) { + message, err := b.chatRepo.GetMessageByIDAndUserID(ctx, id, user.ID) + if err != nil { + return nil, err + } + feedback := message.Feedback + if feedback == nil { + feedback = &model.ChatMessageFeedback{ + ChatMessageID: message.ID, + UserID: user.ID, + } + } + feedback.UpVotes = vote + if err = b.chatRepo.MessageFeedback(ctx, feedback); err != nil { + return nil, err + } + return feedback, nil +} + +func (b *chatBL) SendMessage(ctx *gin.Context, user *model.User, param *parameters.CreateChatMessageRequest) (*responder.Manager, error) { + chatSession, err := b.chatRepo.GetSessionByID(ctx.Request.Context(), user.ID, param.ChatSessionID.IntPart()) + if err != nil { + return nil, err + } + em, err := b.embeddingModelRepo.GetDefault(ctx.Request.Context(), user.TenantID) + if err != nil { + zap.S().Errorf(err.Error()) + em = &model.EmbeddingModel{ + ModelID: b.cfg.DefaultEmbeddingModel, + } + } + message := model.ChatMessage{ + ChatSessionID: chatSession.ID, + Message: param.Message, + MessageType: model.MessageTypeUser, + TimeSent: time.Now().UTC(), + } + noLLM := chatSession.Persona == nil + if err = b.chatRepo.SendMessage(ctx.Request.Context(), &message); err != nil { + return nil, err + } + aiClient := b.aiBuilder.New(chatSession.Persona.LLM) + resp := responder.NewManager( + responder.NewAIResponder(aiClient, b.chatRepo, + b.embedding, b.milvusClinet, b.docRepo, em.ModelID), + ) + + go resp.Send(ctx, user, noLLM, &message, chatSession.Persona) + return resp, nil +} + +func (b *chatBL) GetSessions(ctx context.Context, user *model.User) ([]*model.ChatSession, error) { + return b.chatRepo.GetSessions(ctx, user.ID) +} + +func (b *chatBL) GetSessionByID(ctx context.Context, user *model.User, id int64) (*model.ChatSession, error) { + result, err := b.chatRepo.GetSessionByID(ctx, user.ID, id) + if err != nil { + return nil, err + } + return result, nil +} + +func (b *chatBL) CreateSession(ctx context.Context, user *model.User, param *parameters.CreateChatSession) (*model.ChatSession, error) { + exists, err := b.personaRepo.IsExists(ctx, param.PersonaID.IntPart(), user.TenantID) + if err != nil { + return nil, err + } + if !exists { + return nil, utils.ErrorBadRequest.New("persona is not exists") + } + session := model.ChatSession{ + UserID: user.ID, + Description: param.Description, + CreationDate: time.Now().UTC(), + PersonaID: param.PersonaID, + OneShot: param.OneShot, + } + if err = b.chatRepo.CreateSession(ctx, &session); err != nil { + return nil, err + } + return &session, nil +} + +func NewChatBL( + cfg *Config, + chatRepo repository.ChatRepository, + personaRepo repository.PersonaRepository, + docRepo repository.DocumentRepository, + embeddingModelRepo repository.EmbeddingModelRepository, + aiBuilder *ai.Builder, + embedding proto.EmbedServiceClient, + milvusClinet storage.MilvusClient, +) ChatBL { + return &chatBL{ + cfg: cfg, + chatRepo: chatRepo, + personaRepo: personaRepo, + docRepo: docRepo, + embeddingModelRepo: embeddingModelRepo, + aiBuilder: aiBuilder, + embedding: embedding, + milvusClinet: milvusClinet, + } +} diff --git a/src/backend/core/bll/connector.go b/src/backend/core/bll/connector.go new file mode 100644 index 00000000..b84e901f --- /dev/null +++ b/src/backend/core/bll/connector.go @@ -0,0 +1,115 @@ +package bll + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "time" +) + +const minRefreshFreq = 3600 + +type ( + ConnectorBL interface { + GetAll(ctx context.Context, user *model.User) ([]*model.Connector, error) + GetByID(ctx context.Context, user *model.User, id int64) (*model.Connector, error) + Create(ctx context.Context, user *model.User, param *parameters.CreateConnectorParam) (*model.Connector, error) + Update(ctx context.Context, id int64, user *model.User, param *parameters.UpdateConnectorParam) (*model.Connector, error) + Archive(ctx context.Context, user *model.User, id int64, restore bool) (*model.Connector, error) + } + connectorBL struct { + connectorRepo repository.ConnectorRepository + //messenger messaging.Client + } +) + +func (c *connectorBL) Archive(ctx context.Context, user *model.User, id int64, restore bool) (*model.Connector, error) { + connector, err := c.connectorRepo.GetByIDAndUser(ctx, user.TenantID, user.ID, id) + if err != nil { + return nil, err + } + if !(connector.UserID == user.ID || user.HasRoles(model.RoleAdmin, model.RoleSuperAdmin)) { + return nil, utils.ErrorPermission.New("permission denied") + } + if !restore { + connector.DeletedDate = pg.NullTime{time.Now().UTC()} + } else { + connector.DeletedDate = pg.NullTime{} + } + connector.LastUpdate = pg.NullTime{time.Now().UTC()} + if err = c.connectorRepo.Update(ctx, connector); err != nil { + return nil, err + } + return connector, nil +} + +func NewConnectorBL(connectorRepo repository.ConnectorRepository) ConnectorBL { + return &connectorBL{connectorRepo: connectorRepo} +} + +func (c *connectorBL) Create(ctx context.Context, user *model.User, param *parameters.CreateConnectorParam) (*model.Connector, error) { + + tenantID := uuid.NullUUID{Valid: false} + if param.Shared { + tenantID.Valid = true + tenantID.UUID = user.TenantID + } + conn := model.Connector{ + Name: param.Name, + Type: model.SourceType(param.Source), + ConnectorSpecificConfig: param.ConnectorSpecificConfig, + RefreshFreq: param.RefreshFreq, + UserID: user.ID, + TenantID: tenantID, + Status: model.ConnectorStatusReadyToProcessed, + CreationDate: time.Now().UTC(), + } + + if conn.RefreshFreq < minRefreshFreq { + conn.RefreshFreq = minRefreshFreq + } + if err := c.connectorRepo.Create(ctx, &conn); err != nil { + return nil, err + } + return &conn, nil +} + +func (c *connectorBL) Update(ctx context.Context, id int64, user *model.User, param *parameters.UpdateConnectorParam) (*model.Connector, error) { + conn, err := c.connectorRepo.GetByIDAndUser(ctx, user.TenantID, user.ID, id) + if err != nil { + return nil, err + } + conn.ConnectorSpecificConfig = param.ConnectorSpecificConfig + conn.Name = param.Name + conn.RefreshFreq = param.RefreshFreq + tenantID := uuid.NullUUID{Valid: false} + if param.Shared { + tenantID.Valid = true + tenantID.UUID = user.TenantID + } + + conn.TenantID = tenantID + conn.LastUpdate = pg.NullTime{time.Now().UTC()} + if param.Status != "" { + conn.Status = param.Status + } + if conn.RefreshFreq < minRefreshFreq { + conn.RefreshFreq = minRefreshFreq + } + if err = c.connectorRepo.Update(ctx, conn); err != nil { + return nil, err + } + return conn, nil +} + +func (c *connectorBL) GetAll(ctx context.Context, user *model.User) ([]*model.Connector, error) { + return c.connectorRepo.GetAllByUser(ctx, user.TenantID, user.ID) +} + +func (c *connectorBL) GetByID(ctx context.Context, user *model.User, id int64) (*model.Connector, error) { + return c.connectorRepo.GetByIDAndUser(ctx, user.TenantID, user.ID, id) +} diff --git a/src/backend/core/bll/document.go b/src/backend/core/bll/document.go new file mode 100644 index 00000000..8ec05ac8 --- /dev/null +++ b/src/backend/core/bll/document.go @@ -0,0 +1,56 @@ +package bll + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "context" + "fmt" + "github.com/google/uuid" + "io" + "time" +) + +type ( + DocumentBL interface { + UploadDocument(ctx context.Context, user *model.User, fileName, contentType string, file io.Reader) (*model.Document, error) + } + documentBL struct { + documentRepo repository.DocumentRepository + minioClient storage.MinIOClient + connectorRepo repository.ConnectorRepository + } +) + +func (b *documentBL) UploadDocument(ctx context.Context, user *model.User, fileName, contentType string, file io.Reader) (*model.Document, error) { + + fileURL, signature, err := b.minioClient.Upload(ctx, model.BucketName(user.TenantID), + fmt.Sprintf("user-%s/%s-%s", user.ID.String(), uuid.New().String(), fileName), contentType, file) + if err != nil { + return nil, err + } + connector, err := b.connectorRepo.GetBySource(ctx, user.TenantID, user.ID, model.SourceTypeFile) + if err != nil { + return nil, err + } + document := &model.Document{ + SourceID: fileURL, + ConnectorID: connector.ID, + URL: fileURL, + Signature: signature, + CreationDate: time.Now().UTC(), + } + if err = b.documentRepo.Create(ctx, document); err != nil { + return nil, err + } + return document, nil +} + +func NewDocumentBL(documentRepo repository.DocumentRepository, + connectorRepo repository.ConnectorRepository, + minioClient storage.MinIOClient) DocumentBL { + return &documentBL{documentRepo: documentRepo, + connectorRepo: connectorRepo, + minioClient: minioClient, + } +} diff --git a/src/backend/core/bll/embedding_model.go b/src/backend/core/bll/embedding_model.go new file mode 100644 index 00000000..b539c7f6 --- /dev/null +++ b/src/backend/core/bll/embedding_model.go @@ -0,0 +1,93 @@ +package bll + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/repository" + "context" + "github.com/go-pg/pg/v10" + "time" +) + +type ( + EmbeddingModelBL interface { + GetAll(ctx context.Context, user *model.User, param *parameters.ArchivedParam) ([]*model.EmbeddingModel, error) + GetByID(ctx context.Context, user *model.User, id int64) (*model.EmbeddingModel, error) + Create(ctx context.Context, user *model.User, em *parameters.EmbeddingModelParam) (*model.EmbeddingModel, error) + Update(ctx context.Context, user *model.User, id int64, em *parameters.EmbeddingModelParam) (*model.EmbeddingModel, error) + Delete(ctx context.Context, user *model.User, id int64) (*model.EmbeddingModel, error) + Restore(ctx context.Context, user *model.User, id int64) (*model.EmbeddingModel, error) + } + embeddingModelBL struct { + emRepo repository.EmbeddingModelRepository + } +) + +func (b *embeddingModelBL) Create(ctx context.Context, user *model.User, em *parameters.EmbeddingModelParam) (*model.EmbeddingModel, error) { + embeddingModel := model.EmbeddingModel{ + TenantID: user.TenantID, + ModelID: em.ModelID, + ModelName: em.ModelName, + ModelDim: em.ModelDim, + URL: em.URL, + IsActive: em.IsActive, + CreationDate: time.Now().UTC(), + } + if err := b.emRepo.Create(ctx, &embeddingModel); err != nil { + return nil, err + } + return &embeddingModel, nil +} + +func (b *embeddingModelBL) Update(ctx context.Context, user *model.User, id int64, em *parameters.EmbeddingModelParam) (*model.EmbeddingModel, error) { + existingEM, err := b.emRepo.GetByID(ctx, user.TenantID, id) + if err != nil { + return nil, err + } + existingEM.ModelID = em.ModelID + existingEM.ModelName = em.ModelName + existingEM.ModelDim = em.ModelDim + existingEM.URL = em.URL + existingEM.IsActive = em.IsActive + existingEM.LastUpdate = pg.NullTime{time.Now().UTC()} + if err = b.emRepo.Update(ctx, existingEM); err != nil { + return nil, err + } + return existingEM, nil +} + +func (b *embeddingModelBL) Delete(ctx context.Context, user *model.User, id int64) (*model.EmbeddingModel, error) { + existingEM, err := b.emRepo.GetByID(ctx, user.TenantID, id) + if err != nil { + return nil, err + } + existingEM.DeletedDate = pg.NullTime{time.Now().UTC()} + if err = b.emRepo.Update(ctx, existingEM); err != nil { + return nil, err + } + return existingEM, nil +} + +func (b *embeddingModelBL) Restore(ctx context.Context, user *model.User, id int64) (*model.EmbeddingModel, error) { + existingEM, err := b.emRepo.GetByID(ctx, user.TenantID, id) + if err != nil { + return nil, err + } + existingEM.DeletedDate = pg.NullTime{time.Time{}} + if err = b.emRepo.Update(ctx, existingEM); err != nil { + return nil, err + } + return existingEM, nil +} + +func (b *embeddingModelBL) GetAll(ctx context.Context, user *model.User, param *parameters.ArchivedParam) ([]*model.EmbeddingModel, error) { + return b.emRepo.GetAll(ctx, user.TenantID, param) +} + +func (b *embeddingModelBL) GetByID(ctx context.Context, user *model.User, id int64) (*model.EmbeddingModel, error) { + return b.emRepo.GetByID(ctx, user.TenantID, id) +} + +func NewEmbeddingModelBL(emRepo repository.EmbeddingModelRepository) EmbeddingModelBL { + return &embeddingModelBL{emRepo: emRepo} +} diff --git a/src/backend/core/bll/module.go b/src/backend/core/bll/module.go new file mode 100644 index 00000000..0efb23d3 --- /dev/null +++ b/src/backend/core/bll/module.go @@ -0,0 +1,29 @@ +package bll + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +type Config struct { + RedirectURL string `env:"REDIRECT_URL"` + DefaultEmbeddingModel string `env:"DEFAULT_EMBEDDING_MODEL" envDefault:"paraphrase-multilingual-mpnet-base-v2"` + DefaultEmbeddingVectorSize int `env:"DEFAULT_EMBEDDING_VECTOR_SIZE" envDefault:"768"` +} + +var BLLModule = fx.Options( + fx.Provide(func() (*Config, error) { + cfg := Config{} + err := utils.ReadConfig(&cfg) + return &cfg, err + }), + fx.Provide( + NewConnectorBL, + NewAuthBL, + NewPersonaBL, + NewChatBL, + NewDocumentBL, + NewEmbeddingModelBL, + NewTenantBL, + ), +) diff --git a/src/backend/core/bll/persona.go b/src/backend/core/bll/persona.go new file mode 100644 index 00000000..1bb87ea0 --- /dev/null +++ b/src/backend/core/bll/persona.go @@ -0,0 +1,145 @@ +package bll + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + "github.com/go-pg/pg/v10" + "time" +) + +type ( + PersonaBL interface { + GetAll(ctx context.Context, user *model.User, archived bool) ([]*model.Persona, error) + GetByID(ctx context.Context, user *model.User, id int64) (*model.Persona, error) + Create(ctx context.Context, user *model.User, param *parameters.PersonaParam) (*model.Persona, error) + Update(ctx context.Context, id int64, user *model.User, param *parameters.PersonaParam) (*model.Persona, error) + Archive(ctx context.Context, user *model.User, id int64, restore bool) (*model.Persona, error) + } + personaBL struct { + personaRepo repository.PersonaRepository + chatRepo repository.ChatRepository + } +) + +func (b *personaBL) Archive(ctx context.Context, user *model.User, id int64, restore bool) (*model.Persona, error) { + if !user.HasRoles(model.RoleSuperAdmin, model.RoleAdmin) { + return nil, utils.ErrorPermission.New("do not have permission") + } + + persona, err := b.personaRepo.GetByID(ctx, id, user.TenantID) + if persona.LLM != nil { + persona.LLM.ApiKey = persona.LLM.MaskApiKey() + } + + if err != nil { + return nil, err + } + if restore { + persona.DeletedDate = pg.NullTime{} + } else { + persona.DeletedDate = pg.NullTime{time.Now().UTC()} + } + persona.LastUpdate = pg.NullTime{time.Now().UTC()} + + if err = b.personaRepo.Archive(ctx, persona); err != nil { + return nil, err + } + return persona, nil +} + +func (b *personaBL) Create(ctx context.Context, user *model.User, param *parameters.PersonaParam) (*model.Persona, error) { + + starterMessages, err := json.Marshal(param.StarterMessages) + if err != nil { + return nil, utils.ErrorBadRequest.Wrap(err, "fail to marshal starter messages") + } + persona := model.Persona{ + Name: param.Name, + DefaultPersona: true, + Description: param.Description, + TenantID: user.TenantID, + IsVisible: true, + StarterMessages: starterMessages, + CreationDate: time.Now().UTC(), + LLM: &model.LLM{ + Name: fmt.Sprintf("%s %s", user.FirstName, param.ModelID), + ModelID: param.ModelID, + TenantID: user.TenantID, + CreationDate: time.Now().UTC(), + Url: param.URL, + ApiKey: param.APIKey, + Endpoint: param.Endpoint, + }, + Prompt: &model.Prompt{ + UserID: user.ID, + Name: param.Name, + Description: param.Description, + SystemPrompt: param.SystemPrompt, + TaskPrompt: param.TaskPrompt, + CreationDate: time.Now().UTC(), + }, + } + if err := b.personaRepo.Create(ctx, &persona); err != nil { + return nil, err + } + return &persona, nil +} + +func (b *personaBL) Update(ctx context.Context, id int64, user *model.User, param *parameters.PersonaParam) (*model.Persona, error) { + persona, err := b.personaRepo.GetByID(ctx, id, user.TenantID) + if err != nil { + return nil, err + } + starterMessages, err := json.Marshal(param.StarterMessages) + if err != nil { + return nil, utils.ErrorBadRequest.Wrap(err, "fail to marshal starter messages") + } + persona.Name = param.Name + persona.Description = param.Description + persona.LastUpdate = pg.NullTime{time.Now().UTC()} + persona.StarterMessages = starterMessages + persona.LLM.Endpoint = param.Endpoint + persona.LLM.ModelID = param.ModelID + // update api key if user updates it. + if persona.LLM.MaskApiKey() != param.APIKey { + persona.LLM.ApiKey = param.APIKey + } + persona.LLM.LastUpdate = pg.NullTime{time.Now().UTC()} + persona.Prompt.Name = param.Name + persona.Prompt.Description = param.Description + persona.Prompt.SystemPrompt = param.SystemPrompt + persona.Prompt.TaskPrompt = param.TaskPrompt + persona.Prompt.LastUpdate = pg.NullTime{time.Now().UTC()} + + if err = b.personaRepo.Update(ctx, persona); err != nil { + return nil, err + } + return persona, nil +} + +func NewPersonaBL(personaRepo repository.PersonaRepository, + chatRepo repository.ChatRepository) PersonaBL { + return &personaBL{ + personaRepo: personaRepo, + chatRepo: chatRepo, + } +} +func (b *personaBL) GetAll(ctx context.Context, user *model.User, archived bool) ([]*model.Persona, error) { + return b.personaRepo.GetAll(ctx, user.TenantID, archived) +} + +func (b *personaBL) GetByID(ctx context.Context, user *model.User, id int64) (*model.Persona, error) { + persona, err := b.personaRepo.GetByID(ctx, id, user.TenantID) + if err != nil { + return nil, err + } + if persona.LLM != nil { + persona.LLM.ApiKey = persona.LLM.MaskApiKey() + } + return persona, nil +} diff --git a/src/backend/core/bll/tenant.go b/src/backend/core/bll/tenant.go new file mode 100644 index 00000000..a8f8b51b --- /dev/null +++ b/src/backend/core/bll/tenant.go @@ -0,0 +1,67 @@ +package bll + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/google/uuid" +) + +type ( + TenantBL interface { + GetUsers(ctx context.Context, user *model.User) ([]*model.User, error) + AddUser(ctx context.Context, user *model.User, email, role string) (*model.User, error) + UpdateUser(ctx context.Context, user *model.User, id uuid.UUID, role string) (*model.User, error) + } + tenantBL struct { + tenantRepo repository.TenantRepository + userRepo repository.UserRepository + } +) + +func (b *tenantBL) GetUsers(ctx context.Context, user *model.User) ([]*model.User, error) { + if len(user.Roles) == 0 || user.Roles[0] == model.RoleUser { + return nil, utils.ErrorPermission.New("access denied") + } + return b.tenantRepo.GetUsers(ctx, user.TenantID) +} + +func NewTenantBL(tenantRepo repository.TenantRepository, userRepo repository.UserRepository) TenantBL { + return &tenantBL{tenantRepo: tenantRepo, userRepo: userRepo} +} + +func (b *tenantBL) AddUser(ctx context.Context, user *model.User, email, role string) (*model.User, error) { + exists, err := b.userRepo.IsUserExists(ctx, email) + if err != nil { + return nil, err + } + if exists { + return nil, utils.ErrorBadRequest.New("user already exists") + } + newUser := &model.User{ + ID: uuid.New(), + TenantID: user.TenantID, + UserName: email, + FirstName: "", + LastName: "", + ExternalID: "", + Roles: model.StringSlice{role}, + } + if err := b.userRepo.Create(ctx, newUser); err != nil { + return nil, err + } + return newUser, nil +} + +func (b *tenantBL) UpdateUser(ctx context.Context, user *model.User, id uuid.UUID, role string) (*model.User, error) { + user, err := b.userRepo.GetByIDAndTenantID(ctx, id, user.TenantID) + if err != nil { + return nil, err + } + user.Roles = model.StringSlice{role} + if err = b.userRepo.Update(ctx, user); err != nil { + return nil, err + } + return user, nil +} diff --git a/src/backend/core/connector/base.go b/src/backend/core/connector/base.go new file mode 100644 index 00000000..776d5203 --- /dev/null +++ b/src/backend/core/connector/base.go @@ -0,0 +1,138 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "golang.org/x/oauth2" + "time" +) + +const () + +type Task interface { + RunConnector(ctx context.Context, data *proto.ConnectorRequest) error + RunSemantic(ctx context.Context, data *proto.SemanticData) error + UpToDate(ctx context.Context) error +} + +type Connector interface { + Execute(ctx context.Context, param map[string]string) chan *Response + PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error + Validate() error +} + +type Base struct { + model *model.Connector + connectorRepo repository.ConnectorRepository + resultCh chan *Response + oauthClient *resty.Client +} +type Response struct { + URL string + SiteMap string + SearchForSitemap bool + Name string + SourceID string + DocumentID int64 + //Content []byte + MimeType string + FileType proto.FileType + Signature string + Content *Content + UpToData bool +} + +// Content defines action for stop content in minio database +type Content struct { + Bucket string + URL string // URL for download + Body []byte // Body raw content for store + AppendContent bool // if true content will be added to existing file on minio +} + +type Builder struct { + connectorRepo repository.ConnectorRepository +} + +type nopConnector struct { + Base +} + +func (n *nopConnector) Validate() error { + return nil +} + +func (n *nopConnector) PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error { + return nil +} + +func (n *nopConnector) Execute(ctx context.Context, param map[string]string) chan *Response { + ch := make(chan *Response) + go func() { + time.Sleep(5 * time.Second) + close(ch) + }() + + return ch +} + +func New(connectorModel *model.Connector, + connectorRepo repository.ConnectorRepository, + oauthURL string) (Connector, error) { + switch connectorModel.Type { + case model.SourceTypeWEB: + return NewWeb(connectorModel) + case model.SourceTypeOneDrive: + return NewOneDrive(connectorModel, connectorRepo, oauthURL) + case model.SourceTypeFile: + return NewFile(connectorModel) + case model.SourceTypeYoutube: + return NewYoutube(connectorModel) + case model.SourceTypeMsTeams: + return NewMSTeams(connectorModel, connectorRepo, oauthURL) + default: + return &nopConnector{}, nil + } +} + +func (b *Base) Config(connector *model.Connector) { + b.model = connector + b.resultCh = make(chan *Response, 10) + return +} + +// refreshToken refresh OAuth token and store credential in database +func (b *Base) refreshToken(token *oauth2.Token) (*oauth2.Token, error) { + if token.Expiry.UTC().After(time.Now().UTC()) { + return nil, nil + } + provider, ok := model.ConnectorAuthProvider[b.model.Type] + if !ok { + return nil, nil + } + + response, err := b.oauthClient.R(). + SetBody(token).Post(fmt.Sprintf("/api/oauth/%s/refresh_token", provider)) + if err = utils.WrapRestyError(response, err); err != nil { + return nil, err + } + var payload struct { + Data *oauth2.Token `json:"data"` + } + + if err = json.Unmarshal(response.Body(), &payload); err != nil { + return nil, fmt.Errorf("failed to unmarshl token: %v : %v", err, response.Error()) + } + b.model.ConnectorSpecificConfig["token"] = payload.Data + if err = b.connectorRepo.Update(context.Background(), b.model); err != nil { + return nil, err + } + return payload.Data, nil +} diff --git a/src/backend/core/connector/file.go b/src/backend/core/connector/file.go new file mode 100644 index 00000000..74f5db66 --- /dev/null +++ b/src/backend/core/connector/file.go @@ -0,0 +1,102 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "fmt" + "github.com/google/uuid" + "go.uber.org/zap" + "time" +) + +type ( + File struct { + Base + param *FileParameters + ctx context.Context + } + FileParameters struct { + FileName string `json:"file_name"` + MIMEType string `json:"mime_type"` + } +) + +func (c *File) Validate() error { + return nil +} + +func (c *File) PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error { + + link := fmt.Sprintf("minio:tenant-%s:%s", c.model.User.EmbeddingModel.TenantID.String(), c.param.FileName) + if len(c.model.Docs) == 0 { + c.model.Docs = append(c.model.Docs, &model.Document{ + SourceID: link, + ConnectorID: c.model.ID, + URL: link, + CreationDate: time.Now().UTC(), + IsExists: true, + }) + } + // ignore file that was analyzed + if c.model.Status == model.ConnectorStatusError || c.model.Status == model.ConnectorStatusSuccess { + return nil + } + c.model.Docs[0].ChunkingSession = uuid.NullUUID{sessionID, true} + return task.RunSemantic(ctx, &proto.SemanticData{ + Url: link, + DocumentId: c.model.Docs[0].ID.IntPart(), + ConnectorId: c.model.ID.IntPart(), + FileType: 0, + CollectionName: c.model.CollectionName(), + ModelName: c.model.User.EmbeddingModel.ModelID, + ModelDimension: int32(c.model.User.EmbeddingModel.ModelDim), + }) +} + +func (c *File) Execute(ctx context.Context, param map[string]string) chan *Response { + // do no used for this source + c.ctx = ctx + go func() { + defer close(c.resultCh) + if c.param == nil || c.param.FileName == "" { + return + } + // check id document already exists + doc, ok := c.Base.model.DocsMap[c.param.FileName] + url := fmt.Sprintf("minio:tenant-%s:%s", c.model.User.EmbeddingModel.TenantID, c.param.FileName) + if !ok { + doc = &model.Document{ + SourceID: url, + ConnectorID: c.model.ID, + URL: url, + Signature: "", + } + c.model.DocsMap[url] = doc + } + doc.IsExists = true + if fileType, ok := model.SupportedMimeTypes[c.param.MIMEType]; ok { + c.resultCh <- &Response{ + URL: url, + SourceID: url, + FileType: fileType, + } + } else { + zap.S().Errorf("Upsupported file type : %s ", c.param.MIMEType) + } + + }() + return c.resultCh +} + +// NewFile creates new instance of file connector. +func NewFile(connector *model.Connector) (Connector, error) { + fileConn := File{} + fileConn.Base.Config(connector) + fileConn.param = &FileParameters{} + if err := connector.ConnectorSpecificConfig.ToStruct(fileConn.param); err != nil { + return nil, err + } + + return &fileConn, nil +} diff --git a/src/backend/core/connector/microsoft-core/drive-model.go b/src/backend/core/connector/microsoft-core/drive-model.go new file mode 100644 index 00000000..7351a343 --- /dev/null +++ b/src/backend/core/connector/microsoft-core/drive-model.go @@ -0,0 +1,44 @@ +package microsoft_core + +import ( + "cognix.ch/api/v2/core/proto" + "time" +) + +type DriveResponse struct { + Value []*DriveChildBody `json:"value"` +} + +type DriveChildBody struct { + MicrosoftGraphDownloadUrl string `json:"@microsoft.graph.downloadUrl"` + MicrosoftGraphDecorator string `json:"@microsoft.graph.Decorator"` + Id string `json:"id"` + LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` + Name string `json:"name"` + WebUrl string `json:"webUrl"` + File *MsFile `json:"file"` + Size int `json:"size"` + Folder *Folder `json:"folder"` +} + +type MsFile struct { + Hashes struct { + QuickXorHash string `json:"quickXorHash"` + } `json:"hashes"` + MimeType string `json:"mimeType"` +} + +type Folder struct { + ChildCount int `json:"childCount"` +} + +type Response struct { + URL string + Name string + SourceID string + ParentID string + DocumentID int64 + MimeType string + FileType proto.FileType + Signature string +} diff --git a/src/backend/core/connector/microsoft-core/drive.go b/src/backend/core/connector/microsoft-core/drive.go new file mode 100644 index 00000000..7e85615b --- /dev/null +++ b/src/backend/core/connector/microsoft-core/drive.go @@ -0,0 +1,275 @@ +package microsoft_core + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "go.uber.org/zap" + "strings" +) + +const ( + DownloadItem = "https://graph.microsoft.com/v1.0/me/drive/items/%s" +) + +type ( + MSDriveParam struct { + Folder string + Recursive bool + } + + MSDrive struct { + client *resty.Client + param *MSDriveParam + folderURL string + baseURL string + callback FileCallback + fileSizeLimit int + sessionID uuid.NullUUID + model *model.Connector + unsupportedType map[string]bool + } + FileCallback func(response *Response) +) + +func NewMSDrive(param *MSDriveParam, + model *model.Connector, + sessionID uuid.NullUUID, + clinet *resty.Client, + baseURL, folderURL string, + callback FileCallback) *MSDrive { + return &MSDrive{ + param: param, + model: model, + sessionID: sessionID, + callback: callback, + folderURL: folderURL, + baseURL: baseURL, + client: clinet, + unsupportedType: make(map[string]bool), + } +} + +func (c *MSDrive) Execute(ctx context.Context, fileSizeLimit int) error { + c.fileSizeLimit = fileSizeLimit + body, err := c.request(ctx, c.baseURL) + if err != nil { + return err + } + if body != nil { + if err = c.handleItems(ctx, "", body.Value); err != nil { + return err + } + } + return nil +} + +func (c *MSDrive) DownloadItem(ctx context.Context, itemID string, fileSizeLimit int) error { + var item DriveChildBody + c.fileSizeLimit = fileSizeLimit + + if err := c.requestAndParse(ctx, fmt.Sprintf(DownloadItem, itemID), &item); err != nil { + return err + } + return c.getFile(&item) +} +func (c *MSDrive) getFile(item *DriveChildBody) error { + // do not process files that size greater than limit + if item.Size > c.fileSizeLimit { + return nil + } + + doc, ok := c.model.DocsMap[item.Id] + fileName := "" + if !ok { + doc = &model.Document{ + SourceID: item.Id, + ConnectorID: c.model.ID, + URL: item.MicrosoftGraphDownloadUrl, + Signature: "", + ChunkingSession: c.sessionID, + } + // build unique filename for store in minio + fileName = utils.StripFileName(c.model.BuildFileName(uuid.New().String() + "-" + item.Name)) + c.model.DocsMap[item.Id] = doc + } else { + // when file was stored in minio URL should be minio:bucket:filename + minioFile := strings.Split(doc.URL, ":") + if len(minioFile) == 3 && minioFile[0] == "minio" { + fileName = minioFile[2] + } + // use previous file name for update file in minio + } + doc.OriginalURL = item.WebUrl + doc.IsExists = true + + // do not process file if hash is not changed and file already stored in vector database + if doc.Signature == item.File.Hashes.QuickXorHash { + return nil + //if doc.Analyzed { + // return nil + //} + //todo need to clarify should I send message to semantic service again + } + doc.ChunkingSession = c.sessionID + doc.Signature = item.File.Hashes.QuickXorHash + payload := &Response{ + URL: item.MicrosoftGraphDownloadUrl, + SourceID: item.Id, + Name: fileName, + DocumentID: doc.ID.IntPart(), + } + payload.MimeType, payload.FileType = c.recognizeFiletype(item) + + // try to recognize type of file by content + + if payload.FileType == proto.FileType_UNKNOWN { + return nil + } + + c.callback(payload) + return nil +} + +func (c *MSDrive) recognizeFiletype(item *DriveChildBody) (string, proto.FileType) { + + //mimeTypeParts := strings.Split(item.File.MimeType, ";") + + //if fileType, ok := model.SupportedMimeTypes[mimeTypeParts[0]]; ok { + // return mimeTypeParts[0], fileType + //} + // recognize fileType by filename extension + fileNameParts := strings.Split(item.Name, ".") + if len(fileNameParts) > 1 { + if _, ok := c.unsupportedType[fileNameParts[len(fileNameParts)-1]]; ok { + return "", proto.FileType_UNKNOWN + } + if mimeType, ok := model.SupportedExtensions[strings.ToUpper(fileNameParts[len(fileNameParts)-1])]; ok { + return mimeType, model.SupportedMimeTypes[mimeType] + } + c.unsupportedType[fileNameParts[len(fileNameParts)-1]] = true + zap.S().Infof("unsupported file %s type %s -- %s", item.Name, fileNameParts[len(fileNameParts)-1], item.File.MimeType) + } + // recognize filetype by content + //response, err := c.client.R(). + // SetDoNotParseResponse(true). + // Get(item.MicrosoftGraphDownloadUrl) + //defer response.RawBody().Close() + //if err == nil && !response.IsError() { + // if mime, err := mimetype.DetectReader(response.RawBody()); err == nil { + // if fileType, ok := model.SupportedMimeTypes[mime.String()]; ok { + // return mime.String(), fileType + // } + // } + //} + //if len(fileNameParts) > 1 { + // c.unsupportedType[fileNameParts[len(fileNameParts)-1]] = true + //} + + return "", proto.FileType_UNKNOWN +} + +func (c *MSDrive) getFolder(ctx context.Context, folder string, id string) error { + body, err := c.request(ctx, fmt.Sprintf(c.folderURL, id)) + if err != nil { + return err + } + return c.handleItems(ctx, folder, body.Value) +} + +func (c *MSDrive) handleItems(ctx context.Context, folder string, items []*DriveChildBody) error { + for _, item := range items { + // read files if user do not configure folder name + // or current folder as a part of configured folder. + if !c.isFolderAnalysing(folder) { + continue + } + //if item.File != nil && (strings.Contains(folder, c.param.Folder) || c.param.Folder == "") { + if item.File != nil && c.isFilesAnalysing(folder) { + if err := c.getFile(item); err != nil { + zap.S().Errorf("Failed to get file with id %s : %s ", item.Id, err.Error()) + continue + } + } + if item.Folder != nil { + // do not scan nested folder if user wants to read dod from single folder + if strings.Contains(folder, c.param.Folder) && !c.param.Recursive { + continue + } + nextFolder := folder + if nextFolder != "" { + nextFolder += "/" + } + if err := c.getFolder(ctx, nextFolder+item.Name, item.Id); err != nil { + zap.S().Errorf("Failed to get folder with id %s : %s ", item.Id, err.Error()) + continue + } + } + + } + return nil +} + +func (c *MSDrive) isFolderAnalysing(current string) bool { + mask := c.param.Folder + if len(current) < len(c.param.Folder) { + mask = c.param.Folder[:len(current)] + } + // if user does not set folder name. scan whole oneDrive or only root if recursive is false + if c.param.Folder == "" { + return len(current) == 0 || c.param.Recursive + } + // verify is current folder is part of folder that user configure for scan + if c.param.Recursive { + return strings.HasPrefix(current+"/", mask+"/") || current == c.param.Folder + } + return strings.HasPrefix(current+"/", mask+"/") && len(current) <= len(c.param.Folder) +} + +func (c *MSDrive) isFilesAnalysing(current string) bool { + mask := c.param.Folder + if len(current) < len(c.param.Folder) { + mask = c.param.Folder[:len(mask)] + } + // if user does not set folder name. scan whole oneDrive or only root if recursive is false + if c.param.Folder == "" { + return len(current) == 0 || c.param.Recursive + + } + + if c.param.Recursive { + // recursive + return strings.HasPrefix(current+"/", mask+"/") || current == c.param.Folder + } + // only one folder + return current == c.param.Folder +} + +func (c *MSDrive) requestAndParse(ctx context.Context, url string, result interface{}) error { + response, err := c.client.R().SetContext(ctx).Get(url) + if err = utils.WrapRestyError(response, err); err != nil { + return err + } + return json.Unmarshal(response.Body(), result) +} + +func (c *MSDrive) request(ctx context.Context, url string) (*DriveResponse, error) { + response, err := c.client.R(). + SetContext(ctx). + Get(url) + if err = utils.WrapRestyError(response, err); err != nil { + zap.S().Error(err.Error()) + return nil, err + } + var body DriveResponse + if err = json.Unmarshal(response.Body(), &body); err != nil { + zap.S().Errorw("unmarshal failed", "error", err) + return nil, err + } + return &body, nil +} diff --git a/src/backend/core/connector/microsoft-core/teams-model.go b/src/backend/core/connector/microsoft-core/teams-model.go new file mode 100644 index 00000000..66987abf --- /dev/null +++ b/src/backend/core/connector/microsoft-core/teams-model.go @@ -0,0 +1,96 @@ +package microsoft_core + +import ( + "github.com/go-pg/pg/v10" + "time" +) + +type ( + Team struct { + Id string `json:"id"` + DisplayName string `json:"displayName"` + Description string `json:"description"` + } + + TeamResponse struct { + Value []*Team `json:"value"` + } + ChannelResponse struct { + Value []*ChannelBody `json:"value"` + } + ChannelBody struct { + Id string `json:"id"` + CreatedDateTime time.Time `json:"createdDateTime"` + DisplayName string `json:"displayName"` + Description string `json:"description"` + } + TeamUser struct { + OdataType string `json:"@odata.type"` + Id string `json:"id"` + DisplayName string `json:"displayName"` + UserIdentityType string `json:"userIdentityType"` + TenantId string `json:"tenantId"` + } + TeamFrom struct { + User *TeamUser `json:"user"` + } + + TeamBody struct { + ContentType string `json:"contentType"` + Content string `json:"content"` + } +) + +type TeamFilesFolder struct { + Id string `json:"id"` +} + +type MessageBody struct { + Id string `json:"id"` + Etag string `json:"etag"` + MessageType string `json:"messageType"` + ReplyToId string `json:"replyToId"` + Subject string `json:"subject"` + CreatedDateTime time.Time `json:"createdDateTime"` + LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` + WebUrl string `json:"webUrl"` + DeletedDateTime pg.NullTime `json:"deletedDateTime"` + From *TeamFrom `json:"from"` + Body *TeamBody `json:"body"` + Attachments []*Attachment `json:"attachments"` +} +type MessageResponse struct { + OdataContext string `json:"@odata.context"` + OdataNextLink string `json:"@odata.nextLink"` + OdataDeltaLink string `json:"@odata.deltaLink"` + Value []*MessageBody `json:"value"` +} +type Attachment struct { + Id string `json:"id"` + ContentType string `json:"contentType"` + ContentUrl string `json:"contentUrl"` + Content interface{} `json:"content"` + Name string `json:"name"` + ThumbnailUrl interface{} `json:"thumbnailUrl"` + TeamsAppId interface{} `json:"teamsAppId"` +} + +type MSTeamsChatResponse struct { + Value []*MSTeamsChatValues `json:"value"` + NexLink string `json:"@odata.nextLink"` +} + +type MSTeamsChatValues struct { + Id string `json:"id"` + Topic string `json:"topic"` + CreatedDateTime time.Time `json:"createdDateTime"` + LastUpdatedDateTime time.Time `json:"lastUpdatedDateTime"` + ChatType string `json:"chatType"` + WebUrl string `json:"webUrl"` + TenantId string `json:"tenantId"` + OnlineMeetingInfo interface{} `json:"onlineMeetingInfo"` + Viewpoint struct { + IsHidden bool `json:"isHidden"` + LastMessageReadDateTime time.Time `json:"lastMessageReadDateTime"` + } `json:"viewpoint"` +} diff --git a/src/backend/core/connector/ms-teams.go b/src/backend/core/connector/ms-teams.go new file mode 100644 index 00000000..8cdbb997 --- /dev/null +++ b/src/backend/core/connector/ms-teams.go @@ -0,0 +1,605 @@ +package connector + +import ( + microsoftcore "cognix.ch/api/v2/core/connector/microsoft-core" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + "github.com/go-pg/pg/v10" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "go.uber.org/zap" + "golang.org/x/oauth2" + "jaytaylor.com/html2text" + "strconv" + "strings" + "time" +) + +const ( + msTeamsChannelsURL = "https://graph.microsoft.com/v1.0/teams/%s/channels" + msTeamsMessagesURL = "https://graph.microsoft.com/v1.0/teams/%s/channels/%s/messages/microsoft.graph.delta()" + //msTeamsMessagesURL = "https://graph.microsoft.com/v1.0/teams/%s/channels/%s/messages" + msTeamRepliesURL = "https://graph.microsoft.com/v1.0/teams/%s/channels/%s/messages/%s/replies" + msTeamsInfoURL = "https://graph.microsoft.com/v1.0/teams" + + msTeamsFilesFolder = "https://graph.microsoft.com/v1.0/teams/%s/channels/%s/filesFolder" + msTeamsFolderContent = "https://graph.microsoft.com/v1.0/groups/%s/drive/items/%s/children" + + msTeamsChats = "https://graph.microsoft.com/v1.0/chats?$top=50" + msTeamsChatMessagesURL = "https://graph.microsoft.com/v1.0/chats/%s/messages?$top=50" + + msTeamsParamTeamID = "team_id" + + messageTemplate = `#%s +##%s +` + + messageTypeMessage = "message" + attachmentContentTypReference = "reference" +) + +/* +https://graph.microsoft.com/v1.0/groups/94100e5f-a30f-433d-965e-bde4e817f62a/team/channels/19:65a0a68789ea4abe97c8eec4d6f43786@thread.tacv2 +https://graph.microsoft.com/v1.0/teams/94100e5f-a30f-433d-965e-bde4e817f62a/channels +https://graph.microsoft.com/v1.0/teams/94100e5f-a30f-433d-965e-bde4e817f62a/channels/19:65a0a68789ea4abe97c8eec4d6f43786@thread.tacv2/messages +https://graph.microsoft.com/v1.0/teams/94100e5f-a30f-433d-965e-bde4e817f62a/channels/19:65a0a68789ea4abe97c8eec4d6f43786@thread.tacv2/messages/ +https://graph.microsoft.com/v1.0/teams/94100e5f-a30f-433d-965e-bde4e817f62a/channels/19:65a0a68789ea4abe97c8eec4d6f43786@thread.tacv2/messages/1718016334912/replies + +https://graph.microsoft.com/v1.0/teams/94100e5f-a30f-433d-965e-bde4e817f62a/channels/19:65a0a68789ea4abe97c8eec4d6f43786@thread.tacv2/messages/1718121958378/replies + +https://graph.microsoft.com/v1.0/drives/01SZITRJYIBUNPFKHAYJCLISV4DXJPIJV4/items/ + +https://graph.microsoft.com/v1.0/drives/b!oxsuyS45_EKmyHYegUv4SmEjVp8sBIFPvH1TNMZJZqPviFyz50UFTqjI-nC6wDfJ + +https://graph.microsoft.com/v1.0/groups/94100e5f-a30f-433d-965e-bde4e817f62a/drive/root/children + +// get drive items for channel +https://graph.microsoft.com/v1.0/teams/94100e5f-a30f-433d-965e-bde4e817f62a/channels/19:65a0a68789ea4abe97c8eec4d6f43786@thread.tacv2/filesFolder +// get files from channel +https://graph.microsoft.com/v1.0/groups/94100e5f-a30f-433d-965e-bde4e817f62a/drive/items/01SZITRJYIBUNPFKHAYJCLISV4DXJPIJV4/children + + /groups/94100e5f-a30f-433d-965e-bde4e817f62a/drive/items/01SZITRJYIBUNPFKHAYJCLISV4DXJPIJV4 +*/ +type ( + MSTeams struct { + Base + param *MSTeamParameters + state *MSTeamState + client *resty.Client + fileSizeLimit int + sessionID uuid.NullUUID + } + MSTeamParameters struct { + Team string `json:"team"` + Channels model.StringSlice `json:"channels"` + AnalyzeChats bool `json:"analyze_chats"` + Token *oauth2.Token `json:"token"` + Files *microsoftcore.MSDriveParam `json:"files"` + } + // MSTeamState store ms team state after each execute + MSTeamState struct { + Channels map[string]*MSTeamChannelState `json:"channels"` + Chats map[string]*MSTeamMessageState `json:"chats"` + } + + MSTeamChannelState struct { + // Link for request changes after last execution + DeltaLink string `json:"delta_link"` + Topics map[string]*MSTeamMessageState `json:"topics"` + } + // MSTeamMessageState store + MSTeamMessageState struct { + LastCreatedDateTime time.Time `json:"last_created_date_time"` + } + MSTeamsResult struct { + PrevLoadTime string + Messages []string + } +) + +func (c *MSTeams) Validate() error { + return nil +} + +func (c *MSTeams) PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error { + params := make(map[string]string) + + if c.param.Team != "" { + teamID, err := c.getTeamID(ctx) + if err != nil { + zap.S().Errorf("Prepare task get teamID : %s ", err.Error()) + return err + } + params[msTeamsParamTeamID] = teamID + } + params[model.ParamSessionID] = sessionID.String() + return task.RunConnector(ctx, &proto.ConnectorRequest{ + Id: c.model.ID.IntPart(), + Params: params, + }) +} + +func (c *MSTeams) Execute(ctx context.Context, param map[string]string) chan *Response { + c.resultCh = make(chan *Response) + var fileSizeLimit int + if size, ok := param[model.ParamFileLimit]; ok { + fileSizeLimit, _ = strconv.Atoi(size) + } + if fileSizeLimit == 0 { + fileSizeLimit = 1 + } + c.fileSizeLimit = fileSizeLimit * model.GB + paramSessionID, _ := param[model.ParamSessionID] + if uuidSessionID, err := uuid.Parse(paramSessionID); err != nil { + c.sessionID = uuid.NullUUID{uuid.New(), true} + } else { + c.sessionID = uuid.NullUUID{uuidSessionID, true} + } + + for _, doc := range c.model.Docs { + if doc.Signature == "" { + // do not delete document with chat history. + doc.IsExists = true + } + } + go func() { + defer close(c.resultCh) + if err := c.execute(ctx, param); err != nil { + zap.S().Errorf("execute %s ", err.Error()) + } + return + }() + return c.resultCh +} +func (c *MSTeams) execute(ctx context.Context, param map[string]string) error { + + if c.param.AnalyzeChats { + msDrive := microsoftcore.NewMSDrive(c.param.Files, + c.model, + c.sessionID, c.client, + "", "", + c.getFile, + ) + if err := c.loadChats(ctx, msDrive, ""); err != nil { + zap.S().Errorf("error loading chats : %s ", err.Error()) + //return fmt.Errorf("load chats : %s", err.Error()) + } + } + + if teamID, ok := param[msTeamsParamTeamID]; ok { + if err := c.loadChannels(ctx, teamID); err != nil { + zap.S().Errorf("error loading channels : %s ", err.Error()) + //return fmt.Errorf("load channels : %s", err.Error()) + } + } + // save current state + zap.S().Infof("save connector state.") + if err := c.model.State.FromStruct(c.state); err == nil { + return c.connectorRepo.Update(ctx, c.model) + } + return nil +} + +func (c *MSTeams) loadChannels(ctx context.Context, teamID string) error { + channelIDs, err := c.getChannel(ctx, teamID) + if err != nil { + return err + } + + // loop by channels + for _, channelID := range channelIDs { + // prepare state for channel + channelState, ok := c.state.Channels[channelID] + if !ok { + channelState = &MSTeamChannelState{ + DeltaLink: "", + Topics: make(map[string]*MSTeamMessageState), + } + c.state.Channels[channelID] = channelState + } + + topics, err := c.getTopicsByChannel(ctx, teamID, channelID) + if err != nil { + return err + } + + // load topics + for _, topic := range topics { + // create unique id for store new messages in new document + sourceID := fmt.Sprintf("%s-%s-%s", channelID, topic.Id, uuid.New().String()) + + replies, err := c.getReplies(ctx, teamID, channelID, topic) + if err != nil { + return err + } + if len(replies.Messages) == 0 { + continue + } + doc := &model.Document{ + SourceID: sourceID, + ConnectorID: c.model.ID, + URL: "", + ChunkingSession: c.sessionID, + Analyzed: false, + CreationDate: time.Now().UTC(), + LastUpdate: pg.NullTime{time.Now().UTC()}, + OriginalURL: topic.WebUrl, + IsExists: true, + } + c.model.DocsMap[sourceID] = doc + + fileName := fmt.Sprintf("%s_%s.md", + strings.ReplaceAll(uuid.New().String(), "-", ""), + strings.ReplaceAll(topic.Subject, " ", "")) + c.resultCh <- &Response{ + URL: doc.URL, + Name: fileName, + SourceID: sourceID, + DocumentID: doc.ID.IntPart(), + MimeType: "plain/text", + FileType: proto.FileType_MD, + Signature: "", + Content: &Content{ + Bucket: model.BucketName(c.model.User.EmbeddingModel.TenantID), + URL: "", + AppendContent: true, + Body: []byte(strings.Join(replies.Messages, "\n")), + }, + UpToData: false, + } + } + + if c.param.Files != nil { + if err = c.loadFiles(ctx, teamID, channelID); err != nil { + return err + } + } + } + return nil +} + +// loadFiles scrap channel files +func (c *MSTeams) loadFiles(ctx context.Context, teamID, channelID string) error { + var folderInfo microsoftcore.TeamFilesFolder + if err := c.requestAndParse(ctx, fmt.Sprintf(msTeamsFilesFolder, teamID, channelID), &folderInfo); err != nil { + return err + } + baseUrl := fmt.Sprintf(msTeamsFolderContent, teamID, folderInfo.Id) + folderURL := fmt.Sprintf(msTeamsFolderContent, teamID, `%s`) + msDrive := microsoftcore.NewMSDrive(c.param.Files, + c.model, + c.sessionID, c.client, + baseUrl, folderURL, + c.getFile, + ) + return msDrive.Execute(ctx, c.fileSizeLimit) + +} + +// getChannel get channels from team +func (c *MSTeams) getChannel(ctx context.Context, teamID string) ([]string, error) { + var channelResp microsoftcore.ChannelResponse + if err := c.requestAndParse(ctx, fmt.Sprintf(msTeamsChannelsURL, teamID), &channelResp); err != nil { + return nil, err + } + var channels []string + for _, channel := range channelResp.Value { + if len(c.param.Channels) == 0 || + c.param.Channels.InArray(channel.DisplayName) { + channels = append(channels, channel.Id) + } + } + if len(channels) == 0 { + return nil, fmt.Errorf("channel not found") + } + return channels, nil +} + +func (c *MSTeams) getReplies(ctx context.Context, teamID, channelID string, msg *microsoftcore.MessageBody) (*MSTeamsResult, error) { + var repliesResp microsoftcore.MessageResponse + err := c.requestAndParse(ctx, fmt.Sprintf(msTeamRepliesURL, teamID, channelID, msg.Id), &repliesResp) + if err != nil { + return nil, err + } + var result MSTeamsResult + var messages []string + + state, ok := c.state.Channels[channelID].Topics[msg.Id] + if !ok { + state = &MSTeamMessageState{} + c.state.Channels[channelID].Topics[msg.Id] = state + + if message := c.buildMDMessage(msg); message != "" { + messages = append(messages, message) + } + } else { + result.PrevLoadTime = state.LastCreatedDateTime.Format("2006-01-02-15-04-05") + } + lastTime := state.LastCreatedDateTime + + for _, repl := range repliesResp.Value { + if state.LastCreatedDateTime.UTC().After(repl.CreatedDateTime.UTC()) || + state.LastCreatedDateTime.UTC().Equal(repl.CreatedDateTime.UTC()) { + // ignore messages that were analyzed before + continue + } + if repl.CreatedDateTime.UTC().After(lastTime.UTC()) { + // store timestamp of last message + lastTime = repl.CreatedDateTime + } + if message := c.buildMDMessage(repl); message != "" { + messages = append(messages, message) + } + + } + result.Messages = messages + state.LastCreatedDateTime = lastTime + return &result, nil +} + +func (c *MSTeams) getTopicsByChannel(ctx context.Context, teamID, channelID string) ([]*microsoftcore.MessageBody, error) { + var messagesResp microsoftcore.MessageResponse + // Get url from state. Load changes from previous scan. + state := c.state.Channels[channelID] + + url := state.DeltaLink + if url == "" { + // Load all history if stored lin is empty + url = fmt.Sprintf(msTeamsMessagesURL, teamID, channelID) + } + + if err := c.requestAndParse(ctx, url, &messagesResp); err != nil { + return nil, err + } + if len(messagesResp.Value) > 0 { + if messagesResp.OdataNextLink != "" { + state.DeltaLink = messagesResp.OdataNextLink + } + if messagesResp.OdataDeltaLink != "" { + state.DeltaLink = messagesResp.OdataDeltaLink + } + } + return messagesResp.Value, nil +} + +// getTeamID get team id for current user +func (c *MSTeams) getTeamID(ctx context.Context) (string, error) { + var team microsoftcore.TeamResponse + + if err := c.requestAndParse(ctx, msTeamsInfoURL, &team); err != nil { + return "", err + } + if len(team.Value) == 0 { + return "", fmt.Errorf("team not found") + } + for _, tm := range team.Value { + if tm.DisplayName == c.param.Team { + return tm.Id, nil + } + } + return "", fmt.Errorf("team not found") +} + +// requestAndParse request graph endpoint and parse result. +func (c *MSTeams) requestAndParse(ctx context.Context, url string, result interface{}) error { + response, err := c.client.R().SetContext(ctx).Get(url) + if err = utils.WrapRestyError(response, err); err != nil { + return err + } + return json.Unmarshal(response.Body(), result) +} + +// getFile callback for receive files +func (c *MSTeams) getFile(payload *microsoftcore.Response) { + response := &Response{ + URL: payload.URL, + Name: payload.Name, + SourceID: payload.SourceID, + DocumentID: payload.DocumentID, + MimeType: payload.MimeType, + FileType: payload.FileType, + Signature: payload.Signature, + Content: &Content{ + Bucket: model.BucketName(c.model.User.EmbeddingModel.TenantID), + URL: payload.URL, + }, + } + c.resultCh <- response +} + +func (c *MSTeams) buildMDMessage(msg *microsoftcore.MessageBody) string { + userName := msg.Subject + if msg.From != nil && msg.From.User != nil { + userName = msg.From.User.DisplayName + } + message := msg.Subject + if msg.Body != nil { + message = msg.Body.Content + if msg.Body.ContentType == "html" { + if m, err := html2text.FromString(message, html2text.Options{ + PrettyTables: true, + }); err != nil { + zap.S().Errorf("error building html message: %v", err) + } else { + message = m + } + } + } + if userName == "" && message == "" { + return "" + } + return fmt.Sprintf(messageTemplate, userName, message) +} + +func (c *MSTeams) loadChats(ctx context.Context, msDrive *microsoftcore.MSDrive, nextLink string) error { + var response microsoftcore.MSTeamsChatResponse + url := nextLink + if url == "" { + url = msTeamsChats + } + if err := c.requestAndParse(ctx, url, &response); err != nil { + return nil + } + for _, chat := range response.Value { + sourceID := fmt.Sprintf("chat:%s", chat.Id) + state, ok := c.state.Chats[chat.Id] + if !ok { + state = &MSTeamMessageState{ + LastCreatedDateTime: time.Time{}, + } + c.state.Chats[chat.Id] = state + } + + result, err := c.loadChatMessages(ctx, msDrive, state, chat.Id, fmt.Sprintf(msTeamsChatMessagesURL, chat.Id)) + if err != nil { + zap.S().Errorf("error loading chat messages: %s", err.Error()) + continue + } + if len(result) == 0 { + continue + } + doc := &model.Document{ + SourceID: sourceID, + ConnectorID: c.model.ID, + URL: "", + ChunkingSession: c.sessionID, + Analyzed: false, + CreationDate: time.Now().UTC(), + LastUpdate: pg.NullTime{time.Now().UTC()}, + OriginalURL: chat.WebUrl, + IsExists: true, + } + c.model.DocsMap[sourceID] = doc + + fileName := utils.StripFileName(fmt.Sprintf("%s_%s.md", uuid.New().String(), chat.Id)) + c.resultCh <- &Response{ + URL: doc.URL, + Name: fileName, + SourceID: sourceID, + DocumentID: doc.ID.IntPart(), + MimeType: "text/markdown", + FileType: proto.FileType_MD, + Signature: "", + Content: &Content{ + Bucket: model.BucketName(c.model.User.EmbeddingModel.TenantID), + URL: "", + AppendContent: true, + Body: []byte(strings.Join(result, "\n")), + }, + UpToData: false, + } + } + if response.NexLink != "" { + return c.loadChats(ctx, msDrive, response.NexLink) + } + return nil +} +func (c *MSTeams) loadChatMessages(ctx context.Context, + msDrive *microsoftcore.MSDrive, + state *MSTeamMessageState, + chatID, url string) ([]string, error) { + var response microsoftcore.MessageResponse + if err := c.requestAndParse(ctx, url, &response); err != nil { + return nil, err + } + lastTime := state.LastCreatedDateTime.UTC() + + var messages []string + + for _, msg := range response.Value { + // do not scan system messages + if msg.MessageType != messageTypeMessage { + continue + } + if state.LastCreatedDateTime.UTC().After(msg.CreatedDateTime.UTC()) || + state.LastCreatedDateTime.UTC().Equal(msg.CreatedDateTime.UTC()) { + // messages in desc order. not needed to process messages that were loaded before. + return messages, nil + } + + // renew newest message time + if lastTime.UTC().Before(msg.CreatedDateTime.UTC()) { + lastTime = msg.CreatedDateTime + } + if message := c.buildMDMessage(msg); message != "" { + messages = append(messages, message) + } + for _, attachment := range msg.Attachments { + if err := c.loadAttachment(ctx, msDrive, attachment); err != nil { + zap.S().Errorf("error loading attachment: %v", err) + } + } + } + + if response.OdataNextLink != "" { + if nested, err := c.loadChatMessages(ctx, msDrive, state, chatID, response.OdataNextLink); err == nil { + messages = append(messages, nested...) + } else { + zap.S().Errorf("error loading nested chat messages: %v", err) + } + + } + state.LastCreatedDateTime = lastTime + return messages, nil +} + +func (c *MSTeams) loadAttachment(ctx context.Context, msDrive *microsoftcore.MSDrive, attachment *microsoftcore.Attachment) error { + + if attachment.ContentType != attachmentContentTypReference { + // do not scrap replies + return nil + } + if err := msDrive.DownloadItem(ctx, attachment.Id, c.fileSizeLimit); err != nil { + zap.S().Errorf("download file %s", err.Error()) + } + return nil +} + +// NewMSTeams creates new instance of MsTeams connector +func NewMSTeams(connector *model.Connector, + connectorRepo repository.ConnectorRepository, + oauthURL string) (Connector, error) { + conn := MSTeams{ + Base: Base{ + connectorRepo: connectorRepo, + oauthClient: resty.New(). + SetTimeout(time.Minute). + SetBaseURL(oauthURL), + }, + param: &MSTeamParameters{}, + state: &MSTeamState{}, + } + conn.Base.Config(connector) + + if err := connector.ConnectorSpecificConfig.ToStruct(conn.param); err != nil { + return nil, err + } + + newToken, err := conn.refreshToken(conn.param.Token) + if err != nil { + return nil, err + } + if newToken != nil { + conn.param.Token = newToken + } + if err = connector.State.ToStruct(conn.state); err != nil { + zap.S().Infof("can not parse state %v", err) + } + if conn.state.Channels == nil { + conn.state.Channels = make(map[string]*MSTeamChannelState) + } + if conn.state.Chats == nil { + conn.state.Chats = make(map[string]*MSTeamMessageState) + } + conn.client = resty.New(). + SetTimeout(time.Minute). + SetHeader(authorizationHeader, fmt.Sprintf("%s %s", + conn.param.Token.TokenType, + conn.param.Token.AccessToken)) + return &conn, nil +} diff --git a/src/backend/core/connector/onedrive.go b/src/backend/core/connector/onedrive.go new file mode 100644 index 00000000..28c0efcc --- /dev/null +++ b/src/backend/core/connector/onedrive.go @@ -0,0 +1,209 @@ +package connector + +import ( + microsoft_core "cognix.ch/api/v2/core/connector/microsoft-core" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "context" + "fmt" + _ "github.com/AzureAD/microsoft-authentication-library-for-go/apps/confidential" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "golang.org/x/oauth2" + "strconv" + "strings" + + //"github.com/go-resty/resty/v2" + "go.uber.org/zap" + "time" +) + +const ( + authorizationHeader = "Authorization" + apiBase = "https://graph.microsoft.com/v2.0" + getFilesURL = "/me/drive/special/%s/children" + getDrive = "https://graph.microsoft.com/v1.0/me/drive/root/children" + getFolderChild = "https://graph.microsoft.com/v1.0/me/drive/items/%s/children" + createSharedLink = "https://graph.microsoft.com/v1.0/me/drive/items/%s/createLink" +) + +type ( + OneDrive struct { + Base + param *OneDriveParameters + ctx context.Context + client *resty.Client + fileSizeLimit int + sessionID uuid.NullUUID + } + OneDriveParameters struct { + microsoft_core.MSDriveParam + Token *oauth2.Token `json:"token"` + } +) + +func (c *OneDrive) Validate() error { + return nil +} + +type GetDriveResponse struct { + Value []*DriveChildBody `json:"value"` +} + +type DriveChildBody struct { + MicrosoftGraphDownloadUrl string `json:"@microsoft.graph.downloadUrl"` + MicrosoftGraphDecorator string `json:"@microsoft.graph.Decorator"` + Id string `json:"id"` + LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` + Name string `json:"name"` + WebUrl string `json:"webUrl"` + File *MsFile `json:"file"` + Size int `json:"size"` + Folder *Folder `json:"folder"` +} + +type MsFile struct { + Hashes struct { + QuickXorHash string `json:"quickXorHash"` + } `json:"hashes"` + MimeType string `json:"mimeType"` +} + +type Folder struct { + ChildCount int `json:"childCount"` +} + +func (c *OneDrive) PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error { + // for one drive always send message to connector + return task.RunConnector(ctx, &proto.ConnectorRequest{ + Id: c.model.ID.IntPart(), + Params: map[string]string{ + model.ParamSessionID: sessionID.String(), + }, + }) +} + +func (c *OneDrive) Execute(ctx context.Context, param map[string]string) chan *Response { + var fileSizeLimit int + if size, ok := param[model.ParamFileLimit]; ok { + fileSizeLimit, _ = strconv.Atoi(size) + } + if fileSizeLimit == 0 { + fileSizeLimit = 1 + } + c.fileSizeLimit = fileSizeLimit * model.GB + + paramSessionID, _ := param[model.ParamSessionID] + if uuidSessionID, err := uuid.Parse(paramSessionID); err != nil { + c.sessionID = uuid.NullUUID{uuid.New(), true} + } else { + c.sessionID = uuid.NullUUID{uuidSessionID, true} + } + + if len(c.model.DocsMap) == 0 { + c.model.DocsMap = make(map[string]*model.Document) + } + msDrive := microsoft_core.NewMSDrive( + &c.param.MSDriveParam, + c.model, + c.sessionID, + c.client, + getDrive, + getFolderChild, + c.getFile, + ) + go func() { + defer close(c.resultCh) + if err := msDrive.Execute(ctx, c.fileSizeLimit); err != nil { + zap.S().Errorf(err.Error()) + } + }() + + return c.resultCh +} + +func (c *OneDrive) getFile(payload *microsoft_core.Response) { + response := &Response{ + URL: payload.URL, + Name: payload.Name, + SourceID: payload.SourceID, + DocumentID: payload.DocumentID, + MimeType: payload.MimeType, + FileType: payload.FileType, + Signature: payload.Signature, + Content: &Content{ + Bucket: model.BucketName(c.model.User.EmbeddingModel.TenantID), + URL: payload.URL, + }, + } + c.resultCh <- response +} + +// NewOneDrive creates new instance of OneDrive connector +func NewOneDrive(connector *model.Connector, + connectorRepo repository.ConnectorRepository, + oauthURL string) (Connector, error) { + conn := OneDrive{ + Base: Base{ + connectorRepo: connectorRepo, + oauthClient: resty.New(). + SetTimeout(time.Minute). + SetBaseURL(oauthURL), + }, + param: &OneDriveParameters{}, + } + + conn.Base.Config(connector) + if err := connector.ConnectorSpecificConfig.ToStruct(conn.param); err != nil { + return nil, err + } + newToken, err := conn.refreshToken(conn.param.Token) + if err != nil { + return nil, err + } + if newToken != nil { + conn.param.Token = newToken + } + + conn.client = resty.New(). + SetTimeout(time.Minute). + SetHeader(authorizationHeader, fmt.Sprintf("%s %s", + conn.param.Token.TokenType, + conn.param.Token.AccessToken)) + return &conn, nil +} +func (c *OneDrive) isFolderAnalysing(current string) bool { + mask := c.param.Folder + if len(current) < len(c.param.Folder) { + mask = c.param.Folder[:len(current)] + } + // if user does not set folder name. scan whole oneDrive or only root if recursive is false + if c.param.Folder == "" { + return len(current) == 0 || c.param.Recursive + } + // verify is current folder is part of folder that user configure for scan + if c.param.Recursive { + return strings.HasPrefix(current+"/", mask+"/") || current == c.param.Folder + } + return strings.HasPrefix(current+"/", mask+"/") && len(current) <= len(c.param.Folder) +} + +func (c *OneDrive) isFilesAnalysing(current string) bool { + mask := c.param.Folder + if len(current) < len(c.param.Folder) { + mask = c.param.Folder[:len(mask)] + } + // if user does not set folder name. scan whole oneDrive or only root if recursive is false + if c.param.Folder == "" { + return len(current) == 0 || c.param.Recursive + + } + + if c.param.Recursive { + // recursive + return strings.HasPrefix(current+"/", mask+"/") || current == c.param.Folder + } + // only one folder + return current == c.param.Folder +} diff --git a/src/backend/core/connector/onedrive/ms-teams.md b/src/backend/core/connector/onedrive/ms-teams.md new file mode 100644 index 00000000..800d49b0 --- /dev/null +++ b/src/backend/core/connector/onedrive/ms-teams.md @@ -0,0 +1,2 @@ +list of chats +https://learn.microsoft.com/en-us/graph/api/chat-list?view=graph-rest-1.0&tabs=http \ No newline at end of file diff --git a/src/backend/core/connector/onedrive/readme.md b/src/backend/core/connector/onedrive/readme.md new file mode 100644 index 00000000..40cec24f --- /dev/null +++ b/src/backend/core/connector/onedrive/readme.md @@ -0,0 +1,44 @@ +## Register an application +You can register an application using the Azure Active Directory admin center, or by using the Microsoft Graph PowerShell SDK. + +### Azure Active Directory admin center +Open a browser and navigate to the Azure Active Directory admin center and login using a personal account (aka: Microsoft Account) or Work or School Account. + +Select Azure Active Directory in the left-hand navigation, then select App registrations under Manage. + +Select New registration. Enter a name for your application, for example, Go Graph Tutorial. + +Set Supported account types as desired. The options are: + +Option Who can sign in? +```azure +Accounts in this organizational directory only Only users in your Microsoft 365 organization +Accounts in any organizational directory Users in any Microsoft 365 organization (work or school accounts) +Accounts in any organizational directory ... and personal Microsoft accounts Users in any Microsoft 365 organization (work or school accounts) and personal Microsoft accounts +``` +Leave Redirect URI empty. + +Select Register. On the application's Overview page, copy the value of the Application (client) ID and save it, you will need it in the next step. If you chose Accounts in this organizational directory only for Supported account types, also copy the Directory (tenant) ID and save it. + +Select Authentication under Manage. +Locate the Advanced settings section and change the +Allow public client flows toggle to Yes, + +then choose Save. + +### Create Azure application + +- Go to the Azure App registrations page. https://aka.ms/AppRegistrations/?referrer=https%3A%2F%2Fdev.onedrive.com +- When prompted, sign in with your account credentials. +- Find My applications and click Add an app. +- Enter your app's name and click Create application. + + +#### open Permission page + - Click Application registration + - Api permission page + - Files.Read.All + - offline_access + - Authentication + - Add platform + - choose web and define redirect URL https://rag.cognix.ch/api/oauth/microsoft/callback \ No newline at end of file diff --git a/src/backend/core/connector/onedrive_test.go b/src/backend/core/connector/onedrive_test.go new file mode 100644 index 00000000..ff0fb836 --- /dev/null +++ b/src/backend/core/connector/onedrive_test.go @@ -0,0 +1,100 @@ +package connector + +import ( + _ "github.com/gabriel-vasile/mimetype" + "github.com/stretchr/testify/assert" + "golang.org/x/oauth2" + "testing" +) + +func TestOneDrive_Folder(t *testing.T) { + + c := OneDrive{ + param: &OneDriveParameters{ + Folder: "", + Recursive: false, + Token: oauth2.Token{}, + }, + } + // root without recursive + c.param.Recursive = false + c.param.Folder = "" + assert.Equal(t, c.isFolderAnalysing(""), true) + assert.Equal(t, c.isFolderAnalysing("folder"), false) + + assert.Equal(t, c.isFilesAnalysing(""), true) + assert.Equal(t, c.isFilesAnalysing("folder"), false) + + // root with recursive + c.param.Recursive = true + c.param.Folder = "" + assert.Equal(t, c.isFolderAnalysing(""), true) + assert.Equal(t, c.isFolderAnalysing("folder"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFilesAnalysing(""), true) + assert.Equal(t, c.isFilesAnalysing("folder"), true) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1"), true) + + // given folder name with recursive + c.param.Recursive = true + c.param.Folder = "folder" + + assert.Equal(t, c.isFolderAnalysing(""), true) + assert.Equal(t, c.isFolderAnalysing("folder"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFolderAnalysing("folder2/chapter1"), false) + assert.Equal(t, c.isFolderAnalysing("docs/folder/chapter1"), false) + + assert.Equal(t, c.isFilesAnalysing(""), false) + assert.Equal(t, c.isFilesAnalysing("folder"), true) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFilesAnalysing("folder2/chapter1"), false) + assert.Equal(t, c.isFilesAnalysing("docs/folder/chapter1"), false) + + // given folder name without recursive + c.param.Folder = "folder" + c.param.Recursive = false + assert.Equal(t, c.isFolderAnalysing(""), true) + assert.Equal(t, c.isFolderAnalysing("folder"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1"), false) + assert.Equal(t, c.isFolderAnalysing("folder2/chapter1"), false) + + assert.Equal(t, c.isFilesAnalysing(""), false) + assert.Equal(t, c.isFilesAnalysing("folder"), true) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1"), false) + assert.Equal(t, c.isFilesAnalysing("folder2/chapter1"), false) + + // given subfolder name without recursive + c.param.Folder = "folder/chapter1" + c.param.Recursive = false + assert.Equal(t, c.isFolderAnalysing(""), true) + assert.Equal(t, c.isFolderAnalysing("folder"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1/page1"), false) + assert.Equal(t, c.isFolderAnalysing("folder/chapter2"), false) + assert.Equal(t, c.isFolderAnalysing("folder2/chapter1"), false) + + assert.Equal(t, c.isFilesAnalysing(""), false) + assert.Equal(t, c.isFilesAnalysing("folder"), false) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1/page1"), false) + assert.Equal(t, c.isFolderAnalysing("folder/chapter2"), false) + assert.Equal(t, c.isFilesAnalysing("folder2/chapter1"), false) + + // given subfolder name with recursive + c.param.Folder = "folder/chapter1" + c.param.Recursive = true + assert.Equal(t, c.isFolderAnalysing(""), true) + assert.Equal(t, c.isFolderAnalysing("folder"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter1/page1"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter2"), false) + assert.Equal(t, c.isFolderAnalysing("folder2/chapter1"), false) + + assert.Equal(t, c.isFilesAnalysing(""), false) + assert.Equal(t, c.isFilesAnalysing("folder"), false) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1"), true) + assert.Equal(t, c.isFilesAnalysing("folder/chapter1/page1"), true) + assert.Equal(t, c.isFolderAnalysing("folder/chapter2"), false) + assert.Equal(t, c.isFilesAnalysing("folder2/chapter1"), false) +} diff --git a/src/backend/core/connector/web.go b/src/backend/core/connector/web.go new file mode 100644 index 00000000..57a23a0b --- /dev/null +++ b/src/backend/core/connector/web.go @@ -0,0 +1,108 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "fmt" + "github.com/google/uuid" +) + +type ( + Web struct { + Base + param *WebParameters + } + WebParameters struct { + URL string `url:"url"` + SiteMap string `json:"site_map"` + SearchForSitemap bool `json:"search_for_sitemap"` + URLRecursive bool `json:"url_recursive"` + } +) + +func (c *Web) Validate() error { + return nil +} + +func (c *Web) PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error { + + // if this connector new we need to run connectorTask for prepare document table + if len(c.model.Docs) == 0 { + doc, ok := c.model.DocsMap[c.param.URL] + if !ok { + doc = &model.Document{ + SourceID: c.param.URL, + ConnectorID: c.Base.model.ID, + URL: c.param.URL, + Signature: "", + ChunkingSession: uuid.NullUUID{sessionID, true}, + OriginalURL: c.param.URL, + } + c.model.Docs = append(c.model.Docs, doc) + } + } + var rootDoc *model.Document + for _, doc := range c.model.Docs { + if !doc.ParentID.Valid { + rootDoc = doc + rootDoc.ChunkingSession = uuid.NullUUID{sessionID, true} + break + } + } + c.model.Docs = append([]*model.Document{}, rootDoc) + + if rootDoc == nil { + return fmt.Errorf("root document not found") + } + + return task.RunSemantic(ctx, &proto.SemanticData{ + Url: c.param.URL, + SiteMap: c.param.SiteMap, + SearchForSitemap: c.param.SearchForSitemap, + UrlRecursive: c.param.URLRecursive, + DocumentId: rootDoc.ID.IntPart(), + ConnectorId: c.model.ID.IntPart(), + FileType: proto.FileType_URL, + CollectionName: c.model.CollectionName(), + ModelName: c.model.User.EmbeddingModel.ModelID, + ModelDimension: int32(c.model.User.EmbeddingModel.ModelDim), + }) +} + +func (c *Web) Execute(ctx context.Context, param map[string]string) chan *Response { + go func() { + doc, ok := c.model.DocsMap[c.param.URL] + if !ok { + doc = &model.Document{ + SourceID: c.param.URL, + ConnectorID: c.Base.model.ID, + URL: c.param.URL, + Signature: "", + } + c.Base.model.DocsMap[c.param.URL] = doc + } + c.resultCh <- &Response{ + URL: c.param.URL, + SourceID: c.param.URL, + SiteMap: c.param.SiteMap, + SearchForSitemap: c.param.SearchForSitemap, + DocumentID: doc.ID.IntPart(), + MimeType: model.MIMEURL, + FileType: proto.FileType_URL, + } + close(c.resultCh) + }() + return c.resultCh +} + +func NewWeb(connector *model.Connector) (Connector, error) { + web := Web{} + web.Base.Config(connector) + web.param = &WebParameters{} + if err := connector.ConnectorSpecificConfig.ToStruct(web.param); err != nil { + return nil, err + } + + return &web, nil +} diff --git a/src/backend/core/connector/web_test.go b/src/backend/core/connector/web_test.go new file mode 100644 index 00000000..b40542c3 --- /dev/null +++ b/src/backend/core/connector/web_test.go @@ -0,0 +1,33 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "context" + "github.com/shopspring/decimal" + "testing" +) + +func TestWeb_Execute(t *testing.T) { + web, err := NewWeb( + &model.Connector{ + ID: decimal.NewFromInt(1), + Name: "web test", + Type: "web", + ConnectorSpecificConfig: model.JSONMap{ + "url1": "https://help.collaboard.app/", + "url2": "https://apidog.com/blog/openapi-specification/", + "url": "https://developer.apple.com/documentation/visionos/improving-accessibility-support-in-your-app", + "url3": "https://developer.mozilla.org/en-US/docs/Learn/HTML/Introduction_to_HTML", + }, + DocsMap: make(map[string]*model.Document), + }) + if err != nil { + t.Log(err.Error()) + t.Fatal(err) + } + conn := web.Execute(context.Background(), nil) + + for res := range conn { + t.Log(res.URL) + } +} diff --git a/src/backend/core/connector/web_tokenaizer.go b/src/backend/core/connector/web_tokenaizer.go new file mode 100644 index 00000000..12716e0e --- /dev/null +++ b/src/backend/core/connector/web_tokenaizer.go @@ -0,0 +1,45 @@ +package connector + +import ( + "bytes" + "fmt" + "github.com/gocolly/colly/v2" + "golang.org/x/net/html" + "io" +) + +var skipTag = map[string]bool{ + "script": true, + "style": true, + "meta": true, + "link": true, + "a": true, + "li": true, + "ui": true, +} + +type webTokenizer struct { +} + +func (c *webTokenizer) tokenizer(r *colly.Response) { + tokenizer := html.NewTokenizer(bytes.NewBuffer(r.Body)) + + for { + tokenType := tokenizer.Next() + token := tokenizer.Token() + if tokenizer.Err() == io.EOF { + break + } + if _, ok := skipTag[token.Data]; ok { + if tokenType == html.StartTagToken { + + } + continue + } + if tokenType.String() == "Text" { + // fmt.Println(tokenType.String(), "", token.Data) + fmt.Println(token.String()) + } + } + fmt.Println("--") +} diff --git a/src/backend/core/connector/youtube.go b/src/backend/core/connector/youtube.go new file mode 100644 index 00000000..2f22a2d9 --- /dev/null +++ b/src/backend/core/connector/youtube.go @@ -0,0 +1,69 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "github.com/google/uuid" +) + +type ( + Youtube struct { + Base + param *YoutubeParameters + } + YoutubeParameters struct { + URL string `url:"url"` + } +) + +func (c *Youtube) Validate() error { + return nil +} + +func (c *Youtube) PrepareTask(ctx context.Context, sessionID uuid.UUID, task Task) error { + if len(c.model.Docs) == 0 { + doc, ok := c.model.DocsMap[c.param.URL] + if !ok { + doc = &model.Document{ + SourceID: c.param.URL, + ConnectorID: c.Base.model.ID, + URL: c.param.URL, + Signature: "", + ChunkingSession: uuid.NullUUID{sessionID, true}, + OriginalURL: c.param.URL, + } + c.model.Docs = append(c.model.Docs, doc) + } + } + // ignore file that was analyzed + if c.model.Status == model.ConnectorStatusError || c.model.Status == model.ConnectorStatusSuccess { + return nil + } + return task.RunSemantic(ctx, &proto.SemanticData{ + Url: c.param.URL, + ConnectorId: c.model.ID.IntPart(), + FileType: proto.FileType_YT, + CollectionName: c.model.CollectionName(), + ModelName: c.model.User.EmbeddingModel.ModelID, + ModelDimension: int32(c.model.User.EmbeddingModel.ModelDim), + }) +} + +func (c *Youtube) Execute(ctx context.Context, param map[string]string) chan *Response { + go func() { + close(c.resultCh) + }() + return c.resultCh +} + +func NewYoutube(connector *model.Connector) (Connector, error) { + youtube := Youtube{} + youtube.Base.Config(connector) + youtube.param = &YoutubeParameters{} + if err := connector.ConnectorSpecificConfig.ToStruct(youtube.param); err != nil { + return nil, err + } + + return &youtube, nil +} diff --git a/src/backend/core/messaging/module.go b/src/backend/core/messaging/module.go new file mode 100644 index 00000000..3bba3598 --- /dev/null +++ b/src/backend/core/messaging/module.go @@ -0,0 +1,73 @@ +package messaging + +import ( + "cognix.ch/api/v2/core/utils" + "context" + proto2 "github.com/golang/protobuf/proto" + "github.com/nats-io/nats.go/jetstream" + "go.uber.org/fx" + "go.uber.org/zap" + "time" +) + +const ( + providerNats = "nats" + providerPulsar = "pulsar" +) + +type ( + Config struct { + Nats *natsConfig + //Pulsar *pulsarConfig + Stream *StreamConfig + } + natsConfig struct { + URL string `env:"NATS_CLIENT_URL,required"` + ConnectTimeout int `env:"NATS_CLIENT_CONNECT_TIMEOUT" envDefault:"3"` + ReconnectTimeout int `env:"NATS_CLIENT_RECONNECT_TIME_WAIT" envDefault:"30"` + MaxReconnectAttempts int `env:"NATS_CLIENT_MAX_RECONNECT_ATTEMPTS" envDefault:"3"` + } + // StreamConfig contains variables for configure streams + StreamConfig struct { + ConnectorStreamName string `env:"NATS_CLIENT_CONNECTOR_STREAM_NAME,required"` + ConnectorStreamSubject string `env:"NATS_CLIENT_CONNECTOR_STREAM_SUBJECT,required"` + SemanticStreamName string `env:"NATS_CLIENT_SEMANTIC_STREAM_NAME,required"` + SemanticStreamSubject string `env:"NATS_CLIENT_SEMANTIC_STREAM_SUBJECT,required"` + WhisperStreamName string `env:"NATS_CLIENT_WHISPER_STREAM_NAME,required"` + WhisperStreamSubject string `env:"NATS_CLIENT_WHISPER_STREAM_SUBJECT,required"` + AckWait int `env:"NATS_CLIENT_CONNECTOR_ACK_WAIT,required"` + MaxDeliver int `env:"NATS_CLIENT_CONNECTOR_MAX_DELIVER,required"` + } + + MessageHandler func(ctx context.Context, msg jetstream.Msg) error + Client interface { + Publish(ctx context.Context, streamName, topic string, body proto2.Message) error + Listen(ctx context.Context, streamName, topic string, handler MessageHandler) error + StreamConfig() *StreamConfig + IsOnline() bool + Close() + } +) + +const ( + reconnectAttempts = 120 + reconnectWaitTime = 5 * time.Second + streamMaxPending = 256 +) + +var NatsModule = fx.Options( + fx.Provide(func() (*Config, error) { + cfg := Config{ + Nats: &natsConfig{}, + Stream: &StreamConfig{}, + } + err := utils.ReadConfig(&cfg) + if err != nil { + zap.S().Errorf(err.Error()) + return nil, err + } + return &cfg, nil + }, + NewClientStream, + ), +) diff --git a/src/backend/core/messaging/nats_stream.go b/src/backend/core/messaging/nats_stream.go new file mode 100644 index 00000000..c831f346 --- /dev/null +++ b/src/backend/core/messaging/nats_stream.go @@ -0,0 +1,126 @@ +package messaging + +import ( + "context" + proto2 "github.com/golang/protobuf/proto" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + _ "github.com/nats-io/nats.go/jetstream" + "go.uber.org/zap" + "sync" + "time" +) + +type clientStream struct { + js jetstream.JetStream + conn *nats.Conn + cancel context.CancelFunc + ctx context.Context + wg *sync.WaitGroup + streamCfg *StreamConfig + ackWait time.Duration +} + +func (c *clientStream) IsOnline() bool { + return c.conn.Status() == nats.CONNECTED +} + +func (c *clientStream) StreamConfig() *StreamConfig { + + return c.streamCfg +} + +func (c *clientStream) Close() { + c.wg.Add(1) + c.cancel() + c.wg.Wait() +} + +func (c *clientStream) Publish(ctx context.Context, streamName, topic string, body proto2.Message) error { + _, err := c.js.CreateOrUpdateStream(context.Background(), jetstream.StreamConfig{ + Name: streamName, + Retention: jetstream.WorkQueuePolicy, + Subjects: []string{topic}, + }) + if err != nil { + zap.S().Errorf("Error creating stream: %s", err.Error()) + return err + } + + message, err := proto2.Marshal(body) + if err != nil { + return err + } + _, err = c.js.Publish(ctx, topic, message) + if err != nil { + return err + } + return nil +} + +func (c *clientStream) Listen(ctx context.Context, streamName, topic string, handler MessageHandler) error { + + stream, err := c.js.CreateOrUpdateStream(context.Background(), jetstream.StreamConfig{ + Name: streamName, + Retention: jetstream.WorkQueuePolicy, + Subjects: []string{topic}, + }) + if err != nil { + zap.S().Errorf("Error creating stream: %s", err.Error()) + return err + } + + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: streamName, + MaxDeliver: c.streamCfg.MaxDeliver, + FilterSubject: topic, + AckPolicy: jetstream.AckExplicitPolicy, + AckWait: c.ackWait, + DeliverPolicy: jetstream.DeliverAllPolicy, + }) + if err != nil { + zap.S().Errorf("Failed to create consumer for subscription %v", err) + } + cons.Consume(func(msg jetstream.Msg) { + msg.InProgress() + if err := handler(ctx, msg); err != nil { + zap.S().Errorf("Error handling message: %s", err.Error()) + } + err := msg.Ack() + if err != nil { + zap.S().Errorf("Error acknowledging message: %s", err.Error()) + } + + }) + <-c.ctx.Done() + c.wg.Done() + return nil +} + +func NewClientStream(cfg *Config) (Client, error) { + conn, err := nats.Connect( + cfg.Nats.URL, + nats.Timeout(time.Duration(cfg.Nats.ConnectTimeout)*time.Second), + nats.ReconnectWait(time.Duration(cfg.Nats.ReconnectTimeout)*time.Second), + nats.MaxReconnects(cfg.Nats.MaxReconnectAttempts), + ) + if err != nil { + zap.S().Errorf("Error connecting to NATS: %s", err.Error()) + return nil, err + } + js, err := jetstream.New(conn) + if err != nil { + zap.S().Errorf("Error connecting to NATS: %s", err.Error()) + return nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + return &clientStream{ + js: js, + ctx: ctx, + conn: conn, + cancel: cancel, + wg: &sync.WaitGroup{}, + streamCfg: cfg.Stream, + ackWait: time.Duration(cfg.Stream.AckWait) * time.Second, + }, nil +} diff --git a/src/backend/core/messaging/nats_stream_test.go b/src/backend/core/messaging/nats_stream_test.go new file mode 100644 index 00000000..ee74d9a2 --- /dev/null +++ b/src/backend/core/messaging/nats_stream_test.go @@ -0,0 +1,29 @@ +package messaging + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestClientStream_Listen(t *testing.T) { + cfg := &natsConfig{ + URL: "localhost:4222", + ConnectorStreamName: "test-1", + } + stClient, err := NewClientStream(cfg) + assert.Nil(t, err) + + go func() { + time.Sleep(time.Second * time.Duration(30)) + t.Error("close client stream") + stClient.Close() + }() + stClient.Listen(context.Background(), model.TopicExecutor, model.SubscriptionExecutor, func(ctx context.Context, msg *proto.Message) error { + t.Log(msg.Body.String()) + return nil + }) +} diff --git a/src/backend/core/model/chat.go b/src/backend/core/model/chat.go new file mode 100644 index 00000000..9c51f1b7 --- /dev/null +++ b/src/backend/core/model/chat.go @@ -0,0 +1,89 @@ +package model + +import ( + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "time" +) + +const ( + MessageTypeUser = "user" + MessageTypeAssistant = "assistant" + MessageTypeSystem = "system" +) + +type ( + ChatSession struct { + tableName struct{} `pg:"chat_sessions"` + ID decimal.Decimal `json:"id,omitempty"` + UserID uuid.UUID `json:"user_id,omitempty"` + Description string `json:"description,omitempty"` + CreationDate time.Time `json:"creation_date,omitempty"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty"` + PersonaID decimal.Decimal `json:"persona_id,omitempty"` + OneShot bool `json:"one_shot,omitempty" pg:",use_zero"` + Messages []*ChatMessage `json:"messages,omitempty" pg:"rel:has-many"` + Persona *Persona `json:"persona,omitempty" pg:"rel:has-one"` + } + + ChatMessage struct { + tableName struct{} `pg:"chat_messages"` + ID decimal.Decimal `json:"id,omitempty"` + ChatSessionID decimal.Decimal `json:"chat_session_id,omitempty"` + Message string `json:"message,omitempty" pg:",use_zero"` + MessageType string `json:"message_type,omitempty"` + TimeSent time.Time `json:"time_sent,omitempty"` + TokenCount int `json:"token_count,omitempty" pg:",use_zero"` + ParentMessageID decimal.Decimal `json:"parent_message,omitempty" pg:"parent_message,use_zero"` + LatestChildMessage int `json:"latest_child_message,omitempty" pg:",use_zero"` + RephrasedQuery string `json:"rephrased_query,omitempty" pg:",use_zero"` + Citations []*DocumentResponse `json:"citations,omitempty" pg:"-"` + Error string `json:"error,omitempty" pg:",use_zero"` + Feedback *ChatMessageFeedback `json:"feedback,omitempty" pg:"rel:has-one,fk:id,join_fk:chat_message_id"` + ParentMessage *ChatMessage `json:"-" pg:"-"` + DocumentPairs []*ChatMessageDocumentPair `json:"-" pg:"rel:has-many,fk:chat_message_id,join_fk:chat_message_id"` + } + + ChatMessageFeedback struct { + tableName struct{} `pg:"chat_message_feedbacks"` + ID decimal.Decimal `json:"id,omitempty"` + ChatMessageID decimal.Decimal `json:"chat_message_id,omitempty"` + UserID uuid.UUID `json:"user_id,omitempty"` + UpVotes bool `json:"up_votes" pg:",use_zero"` + Feedback string `json:"feedback,omitempty" pg:",use_zero"` + } + // ChatMessageDocumentPair struct represent data from table chat_message_document_pairs + // what documents were found for each message. + ChatMessageDocumentPair struct { + tableName struct{} `pg:"chat_message_document_pairs"` + ID decimal.Decimal `json:"id"` + ChatMessageID decimal.Decimal `json:"chat_message_id" pg:",use_zero"` + DocumentID decimal.Decimal `json:"document_id" pg:",use_zero"` + Document *Document `json:"document" pg:"rel:has-one"` + } +) + +var _ pg.AfterSelectHook = (*ChatMessage)(nil) + +func (c *ChatMessage) AfterSelect(ctx context.Context) error { + for _, dp := range c.DocumentPairs { + if dp.Document == nil { + continue + } + doc := &DocumentResponse{ + ID: dp.DocumentID, + MessageID: dp.ChatMessageID, + Link: dp.Document.OriginalURL, + DocumentID: dp.Document.SourceID, + } + if !dp.Document.LastUpdate.IsZero() { + doc.UpdatedDate = dp.Document.LastUpdate.Time + } else { + doc.UpdatedDate = dp.Document.CreationDate + } + c.Citations = append(c.Citations, doc) + } + return nil +} diff --git a/src/backend/core/model/connector.go b/src/backend/core/model/connector.go new file mode 100644 index 00000000..0de7a571 --- /dev/null +++ b/src/backend/core/model/connector.go @@ -0,0 +1,64 @@ +package model + +import ( + "fmt" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "strings" + "time" +) + +const ( + CollectionTenant = "tenant_%s" + CollectionUser = "user_%s" + + ConnectorStatusReadyToProcessed = "READY_TO_PROCESS" + ConnectorStatusPending = "PENDING" + ConnectorStatusWorking = "PROCESSING" + ConnectorStatusSuccess = "COMPLETED_SUCCESSFULLY" + ConnectorStatusError = "COMPLETED_WITH_ERRORS" + ConnectorStatusDisabled = "DISABLED" + ConnectorStatusUnableProcess = "UNABLE_TO_PROCESS" +) + +type Connector struct { + tableName struct{} `pg:"connectors"` + ID decimal.Decimal `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type SourceType `json:"source,omitempty" pg:"type"` + ConnectorSpecificConfig JSONMap `json:"connector_specific_config,omitempty"` + RefreshFreq int `json:"refresh_freq,omitempty"` + UserID uuid.UUID `json:"user_id,omitempty"` + TenantID uuid.NullUUID `json:"tenant_id,omitempty"` + LastSuccessfulAnalyzed pg.NullTime `json:"last_successful_analysis,omitempty" pg:",use_zero"` + Status string `json:"status,omitempty"` + TotalDocsAnalyzed int `json:"total_docs_indexed" pg:",use_zero"` + CreationDate time.Time `json:"creation_date,omitempty"` + LastUpdate pg.NullTime `json:"last_update,omitempty" pg:",use_zero"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` + State JSONMap `json:"-"` + Docs []*Document `json:"docs,omitempty" pg:"rel:has-many"` + DocsMap map[string]*Document `json:"docs_map,omitempty" pg:"-"` + User *User `json:"-" pg:"rel:has-one,fk:user_id"` +} + +func (c *Connector) CollectionName() string { + return CollectionName(c.UserID, c.TenantID) +} +func (c *Connector) BuildFileName(filename string) string { + if c.TenantID.Valid { + return fmt.Sprintf("user-%s/%s", c.UserID.String(), filename) + } + return filename +} +func CollectionName(userID uuid.UUID, tenantID uuid.NullUUID) string { + if tenantID.Valid { + return strings.ReplaceAll(fmt.Sprintf(CollectionTenant, tenantID.UUID.String()), "-", "") + } + return strings.ReplaceAll(fmt.Sprintf(CollectionUser, userID.String()), "-", "") +} + +func BucketName(tenantID uuid.UUID) string { + return fmt.Sprintf("tenant-%s", tenantID.String()) +} diff --git a/src/backend/core/model/document.go b/src/backend/core/model/document.go new file mode 100644 index 00000000..f1bddb17 --- /dev/null +++ b/src/backend/core/model/document.go @@ -0,0 +1,33 @@ +package model + +import ( + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "time" +) + +type Document struct { + tableName struct{} `pg:"documents"` + ID decimal.Decimal `json:"id,omitempty"` + ParentID decimal.NullDecimal `json:"parent_id,omitempty" pg:",use_zero"` + SourceID string `json:"source_id,omitempty"` + ConnectorID decimal.Decimal `json:"connector_id,omitempty"` + URL string `json:"url,omitempty" pg:"url"` + Signature string `json:"signature,omitempty" pg:",use_zero"` + ChunkingSession uuid.NullUUID `json:"chunking_session,omitempty" pg:",use_zero"` + Analyzed bool `json:"analyzed" pg:",use_zero"` + CreationDate time.Time `json:"creation_date,omitempty"` + LastUpdate pg.NullTime `json:"last_update,omitempty" pg:",use_zero"` + OriginalURL string `json:"original_url,omitempty" pg:",use_zero"` + IsExists bool `json:"-" pg:"-"` +} + +type DocumentResponse struct { + ID decimal.Decimal `json:"id,omitempty"` + MessageID decimal.Decimal `json:"message_id,omitempty"` + Link string `json:"link,omitempty"` + DocumentID string `json:"document_id,omitempty"` + Content string `json:"content,omitempty"` + UpdatedDate time.Time `json:"updated_date,omitempty"` +} diff --git a/src/backend/core/model/embedding.go b/src/backend/core/model/embedding.go new file mode 100644 index 00000000..a7fad372 --- /dev/null +++ b/src/backend/core/model/embedding.go @@ -0,0 +1,4 @@ +package model + +type Embedding struct { +} diff --git a/src/backend/core/model/embedding_model.go b/src/backend/core/model/embedding_model.go new file mode 100644 index 00000000..2d9a2fbf --- /dev/null +++ b/src/backend/core/model/embedding_model.go @@ -0,0 +1,23 @@ +package model + +import ( + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "time" +) + +type EmbeddingModel struct { + tableName struct{} `pg:"embedding_models"` + + ID decimal.Decimal `json:"id,omitempty"` + TenantID uuid.UUID `json:"tenant_id,omitempty"` + ModelID string `json:"model_id,omitempty"` + ModelName string `json:"model_name,omitempty"` + ModelDim int `json:"model_dim,omitempty" pg:",use_zero"` + URL string `json:"url,omitempty"` + IsActive bool `json:"is_active,omitempty" pg:",use_zero"` + CreationDate time.Time `json:"creation_date,omitempty"` + LastUpdate pg.NullTime `json:"last_update,omitempty" pg:",use_zero"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` +} diff --git a/src/backend/core/model/llm.go b/src/backend/core/model/llm.go new file mode 100644 index 00000000..341dbd86 --- /dev/null +++ b/src/backend/core/model/llm.go @@ -0,0 +1,30 @@ +package model + +import ( + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "time" +) + +type LLM struct { + tableName struct{} `pg:"llms"` + ID decimal.Decimal `json:"id,omitempty"` + Name string `json:"name,omitempty"` + ModelID string `json:"model_id,omitempty"` + TenantID uuid.UUID `json:"tenant_id,omitempty"` + Url string `json:"url,omitempty" pg:",use_zero"` + ApiKey string `json:"api_key"` + Endpoint string `json:"endpoint,omitempty"` + CreationDate time.Time `json:"creation_date,omitempty"` + LastUpdate pg.NullTime `json:"last_update,omitempty" pg:",use_zero"` + + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` +} + +func (l *LLM) MaskApiKey() string { + if len(l.ApiKey) < 10 { + return "***" + } + return string(l.ApiKey[:4]) + "***" + l.ApiKey[len(l.ApiKey)-4:] +} diff --git a/src/backend/core/model/mime_type.go b/src/backend/core/model/mime_type.go new file mode 100644 index 00000000..2ca630a6 --- /dev/null +++ b/src/backend/core/model/mime_type.go @@ -0,0 +1,70 @@ +package model + +import "cognix.ch/api/v2/core/proto" + +const ( + MIMEURL = "url" +) + +var SupportedMimeTypes = map[string]proto.FileType{ + MIMEURL: proto.FileType_URL, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": proto.FileType_XLSX, + "application/vnd.ms-excel": proto.FileType_XLS, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": proto.FileType_DOCX, + "application/msword": proto.FileType_DOC, + "application/pdf": proto.FileType_PDF, + "text/plain": proto.FileType_TXT, + "application/vnd.openxmlformats-officedocument.presentationml.presentation": proto.FileType_PPTX, + "application/vnd.ms-powerpoint": proto.FileType_PPT, + "application/vnd.ms-xpsdocument": proto.FileType_XPS, + "application/oxps": proto.FileType_XPS, + "application/epub+zip": proto.FileType_EPUB, + "application/hwp+zip": proto.FileType_HWPX, + "text/markdown": proto.FileType_MD, + "application/x-mobipocket-ebook": proto.FileType_MOBI, + "application/fb2": proto.FileType_FB2, + "audio/mpeg": proto.FileType_MP3, + "video/mpeg": proto.FileType_MPEG, + "video/mp4": proto.FileType_MP4, + "video/mpga": proto.FileType_MPGA, + "audio/wav": proto.FileType_WAV, + "video/webm": proto.FileType_WEBM, + "video/mov": proto.FileType_MOV, + "video/m4a": proto.FileType_M4A, +} + +var SupportedExtensions = map[string]string{ + "PDF": "application/pdf", + "XLSX": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "XLS": "application/vnd.ms-excel", + "DOCX": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "DOC": "application/msword", + "PPT": "application/vnd.ms-powerpoint", + "PPTX": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "MD": "text/markdown", + "HWPX": "application/hwp+zip", + "MOBI": "application/x-mobipocket-ebook", + "FB2": "application/fb2", + "MP3": "audio/mpeg", + "MPEG": "video/mpeg", + "MP4": "video/mp4", + "MPGA": "video/mpga", + "WAV": "audio/wav", + "WEBM": "video/webm", + "MOV": "video/mov", + "M4A": "video/m4a", + "TXT": "text/plain", + "XPS": "application/vnd.ms-xpsdocument", + "EPUB": "application/epub+zip", +} + +var WhisperFileTypes = map[proto.FileType]bool{ + proto.FileType_MP3: true, + proto.FileType_MP4: true, + proto.FileType_MPEG: true, + proto.FileType_MPGA: true, + proto.FileType_M4A: true, + proto.FileType_WAV: true, + proto.FileType_WEBM: true, + proto.FileType_MOV: true, +} diff --git a/src/backend/core/model/oauth.go b/src/backend/core/model/oauth.go new file mode 100644 index 00000000..af869d36 --- /dev/null +++ b/src/backend/core/model/oauth.go @@ -0,0 +1,14 @@ +package model + +const ( + ProviderCustom OAuthProvider = "custom" + ProviderMicrosoft OAuthProvider = "microsoft" +) + +var ConnectorAuthProvider = map[SourceType]OAuthProvider{ + SourceTypeOneDrive: ProviderMicrosoft, + SourceTypeMsTeams: ProviderMicrosoft, +} + +// OAuthProvider represents enum for oauth providers +type OAuthProvider string diff --git a/src/backend/core/model/persona.go b/src/backend/core/model/persona.go new file mode 100644 index 00000000..366c85e5 --- /dev/null +++ b/src/backend/core/model/persona.go @@ -0,0 +1,28 @@ +package model + +import ( + "encoding/json" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "time" +) + +type Persona struct { + tableName struct{} `pg:"personas"` + ID decimal.Decimal `json:"id,omitempty"` + Name string `json:"name,omitempty"` + LlmID decimal.Decimal `json:"llm_id,omitempty"` + DefaultPersona bool `json:"default_persona,omitempty" pg:",use_zero"` + Description string `json:"description,omitempty" pg:",use_zero"` + TenantID uuid.UUID `json:"tenant_id,omitempty"` + IsVisible bool `json:"is_visible,omitempty" pg:",use_zero"` + DisplayPriority int `json:"display_priority,omitempty"` + StarterMessages json.RawMessage `json:"starter_messages,omitempty" pg:",use_zero"` + LLM *LLM `json:"llm,omitempty" pg:"rel:has-one"` + Prompt *Prompt `json:"prompt,omitempty" pg:"rel:has-one,fk:id,join_fk:persona_id"` + CreationDate time.Time `json:"creation_date,omitempty"` + LastUpdate pg.NullTime `json:"last_update,omitempty" pg:",use_zero"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` + ChatSessions []*ChatSession `json:"chat_sessions,omitempty" pg:"rel:has-many,fk:id,join_fk:persona_id""` +} diff --git a/src/backend/core/model/prompt.go b/src/backend/core/model/prompt.go new file mode 100644 index 00000000..5859b718 --- /dev/null +++ b/src/backend/core/model/prompt.go @@ -0,0 +1,22 @@ +package model + +import ( + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "time" +) + +type Prompt struct { + tableName struct{} `pg:"prompts"` + ID decimal.Decimal `json:"id,omitempty"` + PersonaID decimal.Decimal `json:"persona_id,omitempty"` + UserID uuid.UUID `json:"user_id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty" pg:",use_zero"` + SystemPrompt string `json:"system_prompt,omitempty" pg:",use_zero"` + TaskPrompt string `json:"task_prompt,omitempty" pg:",use_zero"` + CreationDate time.Time `json:"creation_date,omitempty"` + LastUpdate pg.NullTime `json:"last_update,omitempty" pg:",use_zero"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` +} diff --git a/src/backend/core/model/source_type.go b/src/backend/core/model/source_type.go new file mode 100644 index 00000000..9dd897be --- /dev/null +++ b/src/backend/core/model/source_type.go @@ -0,0 +1,76 @@ +package model + +const ( + SourceTypeIngestionApi SourceType = "ingestion_api" + SourceTypeSlack SourceType = "slack" + SourceTypeWEB SourceType = "web" + SourceTypeGoogleDrive SourceType = "google_drive" + SourceTypeGMAIL SourceType = "gmail" + SourceTypeRequesttracker SourceType = "requesttracker" + SourceTypeGithub SourceType = "github" + SourceTypeGitlab SourceType = "gitlab" + SourceTypeGuru SourceType = "guru" + SourceTypeBookstack SourceType = "bookstack" + SourceTypeConfluence SourceType = "confluence" + SourceTypeSlab SourceType = "slab" + SourceTypeJira SourceType = "jira" + SourceTypeProductboard SourceType = "productboard" + SourceTypeFile SourceType = "file" + SourceTypeNotion SourceType = "notion" + SourceTypeZulip SourceType = "zulip" + SourceTypeLinear SourceType = "linear" + SourceTypeHubspot SourceType = "hubspot" + SourceTypeDocument360 SourceType = "document360" + SourceTypeGong SourceType = "gong" + SourceTypeGoogleSites SourceType = "google_sites" + SourceTypeZendesk SourceType = "zendesk" + SourceTypeLoopio SourceType = "loopio" + SourceTypeSharepoint SourceType = "sharepoint" + SourceTypeOneDrive SourceType = "one-drive" + SourceTypeMsTeams SourceType = "msteams" + SourceTypeYoutube SourceType = "youtube" +) + +type ( + SourceType string + SourceTypeDescription struct { + ID SourceType `json:"id"` + Name string `json:"name"` + IsImplemented bool `json:"isImplemented"` + } +) + +var ( + sourceTypeFileDescription = SourceTypeDescription{SourceTypeFile, "File", true} + sourceTypeWEBDescription = SourceTypeDescription{SourceTypeWEB, "Web", true} + sourceTypeSlackDescription = SourceTypeDescription{SourceTypeSlack, "Slack", false} + sourceTypeGoogleDriveDescription = SourceTypeDescription{SourceTypeGoogleDrive, "Google Drive", false} + sourceTypeGmailDescription = SourceTypeDescription{SourceTypeGMAIL, "Gmail", false} + sourceTypeSharepointDescription = SourceTypeDescription{SourceTypeSharepoint, "Sharepoint", false} + sourceTypeOneDriveDescription = SourceTypeDescription{SourceTypeOneDrive, "OneDrive", true} + sourceTypeMsTeamsDescription = SourceTypeDescription{SourceTypeMsTeams, "Teams", true} + sourceTypeYouTubeDescription = SourceTypeDescription{SourceTypeYoutube, "Youtube", true} +) +var AllSourceTypes = map[SourceType]*SourceTypeDescription{ + SourceTypeFile: &sourceTypeFileDescription, + SourceTypeWEB: &sourceTypeWEBDescription, + SourceTypeSlack: &sourceTypeSlackDescription, + SourceTypeGoogleDrive: &sourceTypeGoogleDriveDescription, + SourceTypeGMAIL: &sourceTypeGmailDescription, + SourceTypeSharepoint: &sourceTypeSharepointDescription, + SourceTypeOneDrive: &sourceTypeOneDriveDescription, + SourceTypeMsTeams: &sourceTypeMsTeamsDescription, + SourceTypeYoutube: &sourceTypeYouTubeDescription, +} + +var SourceTypesList = []*SourceTypeDescription{ + &sourceTypeFileDescription, + &sourceTypeWEBDescription, + &sourceTypeSlackDescription, + &sourceTypeGoogleDriveDescription, + &sourceTypeGmailDescription, + &sourceTypeSharepointDescription, + &sourceTypeOneDriveDescription, + &sourceTypeMsTeamsDescription, + &sourceTypeYouTubeDescription, +} diff --git a/src/backend/core/model/tenant.go b/src/backend/core/model/tenant.go new file mode 100644 index 00000000..934db147 --- /dev/null +++ b/src/backend/core/model/tenant.go @@ -0,0 +1,10 @@ +package model + +import "github.com/google/uuid" + +type Tenant struct { + tableName struct{} `pg:"tenants"` + ID uuid.UUID `json:"id` + Name string `json:"name"` + Configuration JSONMap `json:"configuration" pg:"type:jsonb"` +} diff --git a/src/backend/core/model/trace.go b/src/backend/core/model/trace.go new file mode 100644 index 00000000..1f45f4f4 --- /dev/null +++ b/src/backend/core/model/trace.go @@ -0,0 +1,8 @@ +package model + +const ( + SpanAttributeConnectorID = "connector-id" + SpanAttributeConnectorSource = "connector-source" + + TracerConnector = "connector" +) diff --git a/src/backend/core/model/types.go b/src/backend/core/model/types.go new file mode 100644 index 00000000..9b64d0a0 --- /dev/null +++ b/src/backend/core/model/types.go @@ -0,0 +1,105 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "github.com/lib/pq" +) + +const ( + GB = 1024 * 1024 * 1024 + ParamFileLimit = "file_limit" + ParamSessionID = "session_id" +) + +type JSONMap map[string]interface{} + +func (j JSONMap) Value() (driver.Value, error) { + if j == nil { + return "{}", nil + } + buf, err := json.Marshal(j) + return string(buf), err +} + +func (j *JSONMap) Scan(src interface{}) error { + source, ok := src.([]byte) + if !ok { + return errors.New("type assertion .([]byte) failed") + } + err := json.Unmarshal(source, j) + if err != nil { + return err + } + if *j == nil { + err = json.Unmarshal([]byte("{}"), j) + if err != nil { + return err + } + } + return nil +} + +func (j *JSONMap) ToStruct(dest interface{}) error { + buf, err := json.Marshal(j) + if err != nil { + return err + } + return json.Unmarshal(buf, dest) +} + +func (j *JSONMap) FromStruct(src interface{}) error { + buf, err := json.Marshal(src) + if err != nil { + return err + } + return json.Unmarshal(buf, j) +} + +// StringSlice is a type that represents string slice. +type StringSlice []string + +// Value implements the driver Valuer interface. +func (t StringSlice) Value() (driver.Value, error) { + return pq.StringArray(t).Value() +} + +// Scan implements the Scanner interface. +func (t *StringSlice) Scan(value interface{}) error { + items := pq.StringArray{} + if err := items.Scan(value); err != nil { + return err + } + *t = []string(items) + return nil +} +func (t StringSlice) InArray(val string) bool { + for _, item := range t { + if item == val { + return true + } + } + return false +} + +type JSON json.RawMessage + +// Scan scan value into Jsonb, implements sql.Scanner interface +func (j *JSON) Scan(value interface{}) error { + bytes, ok := value.([]byte) + if !ok { + return errors.New(fmt.Sprint("Failed to unmarshal JSONB value:", value)) + } + *j = bytes + return nil +} + +// Value return json value, implement driver.Valuer interface +func (j JSON) Value() (driver.Value, error) { + if len(j) == 0 { + return nil, nil + } + return j, nil +} diff --git a/src/backend/core/model/user.go b/src/backend/core/model/user.go new file mode 100644 index 00000000..9929e699 --- /dev/null +++ b/src/backend/core/model/user.go @@ -0,0 +1,37 @@ +package model + +import "github.com/google/uuid" + +const ( + RoleUser = "user" + RoleAdmin = "admin" + RoleSuperAdmin = "super_admin" +) + +type Defaults struct { + EmbeddingModel *EmbeddingModel +} + +type User struct { + ID uuid.UUID `json:"id,omitempty"` + TenantID uuid.UUID `json:"tenant_id,omitempty"` + UserName string `json:"user_name,omitempty"` + FirstName string `json:"first_name,omitempty"` + LastName string `json:"last_name,omitempty"` + ExternalID string `json:"-"` + Roles StringSlice `json:"roles,omitempty" pg:",array"` + Tenant *Tenant `json:"tenant,omitempty" pg:"rel:has-one"` + EmbeddingModel *EmbeddingModel `json:"-" pg:"rel:has-one,fk:tenant_id,join_fk:tenant_id"` + Defaults *Defaults `json:"-" pg:"-"` +} + +func (u *User) HasRoles(role ...string) bool { + for _, r := range role { + for _, ur := range u.Roles { + if ur == r { + return true + } + } + } + return false +} diff --git a/src/backend/core/oauth/client.go b/src/backend/core/oauth/client.go new file mode 100644 index 00000000..d94d2094 --- /dev/null +++ b/src/backend/core/oauth/client.go @@ -0,0 +1,53 @@ +package oauth + +import ( + "context" + "fmt" + "golang.org/x/oauth2" +) + +const ( + LoginState = "login" + SignUpState = "signUp" + InviteState = "invite" + + ProviderGoogle = "google" + ProviderMicrosoft = "microsoft" +) + +var Providers = map[string]bool{ + ProviderMicrosoft: true, +} + +type ( + SignInConfig struct { + State string + StateCookieName string + URL string + } + + IdentityResponse struct { + ID string `json:"id"` + Email string `json:"email"` + Name string `json:"name"` + GivenName string `json:"given_name"` + FamilyName string `json:"family_name"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + Token *oauth2.Token `json:"token,omitempty"` + } + + Proxy interface { + GetAuthURL(ctx context.Context, redirectUrl, state string) (string, error) + ExchangeCode(ctx context.Context, code string) (*IdentityResponse, error) + RefreshToken(token *oauth2.Token) (*oauth2.Token, error) + } +) + +func NewProvider(name string, cfg *Config) (Proxy, error) { + switch name { + case ProviderMicrosoft: + return NewMicrosoft(cfg.Microsoft), nil + } + return nil, fmt.Errorf("unknown provider: %s", name) +} diff --git a/src/backend/core/oauth/google.go b/src/backend/core/oauth/google.go new file mode 100644 index 00000000..1e200fb2 --- /dev/null +++ b/src/backend/core/oauth/google.go @@ -0,0 +1,76 @@ +package oauth + +import ( + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + "github.com/go-resty/resty/v2" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "time" +) + +const ( + StateNameGoogle = "state" + CodeNameGoogle = "code" + oauthGoogleUrlAPI = "https://www.googleapis.com/oauth2/v2/userinfo?access_token=" +) + +type GoogleConfig struct { + GoogleClientID string `env:"GOOGLE_CLIENT_ID"` + GoogleSecret string `env:"GOOGLE_SECRET"` +} + +type googleProvider struct { + config *oauth2.Config + httpClient *resty.Client +} + +// NewGoogleProvider create new implementation of google oAuth client +func NewGoogleProvider(cfg *GoogleConfig, redirectURL string) Proxy { + return &googleProvider{ + httpClient: resty.New().SetTimeout(time.Minute), + config: &oauth2.Config{ + ClientID: cfg.GoogleClientID, + ClientSecret: cfg.GoogleSecret, + Endpoint: google.Endpoint, + RedirectURL: fmt.Sprintf("%s/google/callback", redirectURL), + Scopes: []string{"https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile"}, + }, + } +} + +func (g *googleProvider) GetAuthURL(ctx context.Context, redirectURL, state string) (string, error) { + g.config.RedirectURL = fmt.Sprintf("%s/google/callback", redirectURL) + + return g.config.AuthCodeURL(state, + oauth2.ApprovalForce), nil +} + +func (g *googleProvider) ExchangeCode(ctx context.Context, code string) (*IdentityResponse, error) { + token, err := g.config.Exchange(ctx, code) + if err != nil { + return nil, utils.Internal.Wrapf(err, "code exchange wrong: %s", err.Error()) + } + response, err := g.httpClient.R().Get(oauthGoogleUrlAPI + token.AccessToken) + if err = utils.WrapRestyError(response, err); err != nil { + return nil, utils.Internal.Newf("failed getting user info: %s", err.Error()) + } + + contents := response.Body() + + var data IdentityResponse + if err = json.Unmarshal(contents, &data); err != nil { + return nil, utils.Internal.Wrapf(err, "can not marshal google response") + } + data.AccessToken = token.AccessToken + data.RefreshToken = token.RefreshToken + + return &data, nil +} + +func (g *googleProvider) RefreshToken(token *oauth2.Token) (*oauth2.Token, error) { + return token, nil +} diff --git a/src/backend/core/oauth/microsoft.go b/src/backend/core/oauth/microsoft.go new file mode 100644 index 00000000..9893fcc5 --- /dev/null +++ b/src/backend/core/oauth/microsoft.go @@ -0,0 +1,121 @@ +package oauth + +import ( + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + "github.com/go-resty/resty/v2" + "golang.org/x/oauth2" + "time" +) + +const ( + microsoftLoginURL = `https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=%s&scope=%s&response_type=code&redirect_uri=%s` + microsoftToken = `https://login.microsoftonline.com/organizations/oauth2/v2.0/token` +) + +const microsoftScope = "offline_access Files.Read.All Sites.ReadWrite.All" + +const teamsScope = "ChannelMessage.Read.All Chat.Read Chat.ReadBasic Team.ReadBasic.All TeamSettings.Read.All ChannelSettings.Read.All Channel.ReadBasic.All Group.Read.All Directory.Read.All" + +type ( + Config struct { + Microsoft *MicrosoftConfig + Google *GoogleConfig + } + + // MicrosoftConfig declare configuration for Microsoft OAuth service + MicrosoftConfig struct { + ClientID string `env:"MICROSOFT_CLIENT_ID,required"` + ClientSecret string `env:"MICROSOFT_CLIENT_SECRET,required"` + RedirectUL string + } + microsoftExchangeCodeRequest struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + Code string `json:"code,omitempty"` + GrantType string `json:"grant_type"` + RefreshToken string `json:"refresh_token,omitempty"` + } + tokenResponse struct { + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` + Scope string `json:"scope"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + } + + // Microsoft implement OAuth authorization for microsoft`s services + Microsoft struct { + cfg *MicrosoftConfig + httpClient *resty.Client + } +) + +func (m *Microsoft) GetAuthURL(ctx context.Context, redirectUrl, state string) (string, error) { + m.cfg.RedirectUL = fmt.Sprintf("%s/api/oauth/microsoft/callback", redirectUrl) + return fmt.Sprintf(microsoftLoginURL, m.cfg.ClientID, microsoftScope+" "+teamsScope, m.cfg.RedirectUL), nil +} + +func (m *Microsoft) ExchangeCode(ctx context.Context, code string) (*IdentityResponse, error) { + + payload := map[string]string{ + "client_id": m.cfg.ClientID, + "client_secret": m.cfg.ClientSecret, + "code": code, + "grant_type": "authorization_code", + "redirect_uri": m.cfg.RedirectUL, + } + var response tokenResponse + resp, err := m.httpClient.R().SetFormData(payload). + Post(microsoftToken) + if err = utils.WrapRestyError(resp, err); err != nil { + return nil, utils.ErrorPermission.Newf("exchange code error: %s", err.Error()) + } + if err = json.Unmarshal(resp.Body(), &response); err != nil { + return nil, err + } + return &IdentityResponse{ + Token: &oauth2.Token{ + AccessToken: response.AccessToken, + TokenType: response.TokenType, + RefreshToken: response.RefreshToken, + Expiry: time.Now().Add(time.Duration(response.ExpiresIn) * time.Second), + }, + }, nil +} + +func (m *Microsoft) RefreshToken(token *oauth2.Token) (*oauth2.Token, error) { + payload := map[string]string{ + "client_id": m.cfg.ClientID, + "client_secret": m.cfg.ClientSecret, + "refresh_token": token.RefreshToken, + "grant_type": "refresh_token", + "redirect_uri": m.cfg.RedirectUL, + } + var response tokenResponse + resp, err := m.httpClient.R().SetFormData(payload). + Post(microsoftToken) + if err = utils.WrapRestyError(resp, err); err != nil { + return nil, utils.ErrorPermission.Newf("exchange code error: %s", err.Error()) + } + if err = json.Unmarshal(resp.Body(), &response); err != nil { + return nil, err + } + return &oauth2.Token{ + AccessToken: response.AccessToken, + TokenType: response.TokenType, + RefreshToken: response.RefreshToken, + Expiry: time.Now().Add(time.Duration(response.ExpiresIn) * time.Second), + }, nil +} + +//urn:ietf:params:oauth:grant-type:jwt-bearer + +func NewMicrosoft(cfg *MicrosoftConfig) Proxy { + return &Microsoft{ + cfg: cfg, + httpClient: resty.New().SetTimeout(time.Minute), + } +} diff --git a/src/backend/core/oauth/microsoft_test.go b/src/backend/core/oauth/microsoft_test.go new file mode 100644 index 00000000..0fca1930 --- /dev/null +++ b/src/backend/core/oauth/microsoft_test.go @@ -0,0 +1,17 @@ +package oauth + +import ( + "cognix.ch/api/v2/core/utils" + "context" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_MicrosoftGetURL(t *testing.T) { + cfg := MicrosoftConfig{} + err := utils.ReadConfig(&cfg) + assert.NoError(t, err) + oauth := NewMicrosoft(&cfg) + url, err := oauth.GetAuthURL(context.Background(), "http://localhost:8080/", "") + t.Log(url) +} diff --git a/src/backend/core/parameters/chat.go b/src/backend/core/parameters/chat.go new file mode 100644 index 00000000..d6657d07 --- /dev/null +++ b/src/backend/core/parameters/chat.go @@ -0,0 +1,75 @@ +package parameters + +import ( + "cognix.ch/api/v2/core/model" + "fmt" + validation "github.com/go-ozzo/ozzo-validation/v4" + "github.com/shopspring/decimal" + "time" +) + +const ( + MessageFeedbackUpvote = "upvote" + MessageFeedbackDownvote = "downvote" +) + +type CreateChatSession struct { + Description string `json:"description"` + PersonaID decimal.Decimal `json:"persona_id"` + OneShot bool `json:"one_shot"` +} + +func (v CreateChatSession) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.PersonaID, validation.Required, + validation.By(func(value interface{}) error { + if v.PersonaID.IsZero() { + return fmt.Errorf("persona_id is zero") + } + return nil + })), + ) +} + +type CreateChatMessageRequest struct { + ChatSessionID decimal.Decimal `json:"chat_session_id,omitempty"` + ParentMessageID decimal.Decimal `json:"parent_message_id,omitempty"` + Message string `json:"message,omitempty"` + PromptID decimal.Decimal `json:"prompt_id,omitempty"` + SearchDocIds []decimal.Decimal `json:"search_doc_ids,omitempty"` + //RetrievalOptions RetrievalDetails `json:"retrieval_options,omitempty"` + QueryOverride string `json:"query_override,omitempty"` + NoAiAnswer bool `json:"no_ai_answer,omitempty"` +} + +func (v CreateChatMessageRequest) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.ChatSessionID, validation.Required), + validation.Field(&v.Message, validation.Required)) +} + +type RetrievalDetails struct { + RunSearch string `json:"run_search,omitempty"` + RealTime bool `json:"real_time,omitempty"` + Filters BaseFilters `json:"filters,omitempty"` + EnableAutoDetectFilters bool `json:"enable_auto_detect_filters,omitempty"` + Offset int `json:"offset,omitempty"` + Limit int `json:"limit,omitempty"` +} +type BaseFilters struct { + SourceType []model.SourceType `json:"source_type,omitempty"` + DocumentSet []string `json:"document_set,omitempty"` + TimeCutoff time.Time `json:"time_cutoff,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +type MessageFeedbackParam struct { + ID decimal.Decimal `json:"id"` + Vote string `json:"vote"` +} + +func (v MessageFeedbackParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.ID, validation.Required), + validation.Field(&v.Vote, validation.Required, validation.In(MessageFeedbackDownvote, MessageFeedbackUpvote))) +} diff --git a/src/backend/core/parameters/docs.go b/src/backend/core/parameters/docs.go new file mode 100644 index 00000000..6fdd4d18 --- /dev/null +++ b/src/backend/core/parameters/docs.go @@ -0,0 +1,19 @@ +package parameters + +import "github.com/shopspring/decimal" + +type DocumentSetParam struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type DocumentSetConnectorPairsParam struct { + DocumentSetID decimal.Decimal `json:"document_set_id"` + ConnectorIDs []decimal.Decimal `json:"connector_ids"` +} + +type DocumentUploadResponse struct { + FileName string `json:"file_name"` + Error string `json:"error"` + Document interface{} `json:"document"` +} diff --git a/src/backend/core/parameters/embedding_model.go b/src/backend/core/parameters/embedding_model.go new file mode 100644 index 00000000..2fa5b673 --- /dev/null +++ b/src/backend/core/parameters/embedding_model.go @@ -0,0 +1,16 @@ +package parameters + +import validation "github.com/go-ozzo/ozzo-validation/v4" + +type EmbeddingModelParam struct { + ModelID string `json:"model_id,omitempty"` + ModelName string `json:"model_name,omitempty"` + ModelDim int `json:"model_dim,omitempty"` + URL string `json:"url,omitempty"` + IsActive bool `json:"is_active,omitempty"` +} + +func (v EmbeddingModelParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.ModelID, validation.Required)) +} diff --git a/src/backend/core/parameters/model.go b/src/backend/core/parameters/model.go new file mode 100644 index 00000000..a458b049 --- /dev/null +++ b/src/backend/core/parameters/model.go @@ -0,0 +1,94 @@ +package parameters + +import ( + "cognix.ch/api/v2/core/model" + "fmt" + validation "github.com/go-ozzo/ozzo-validation/v4" + "github.com/go-ozzo/ozzo-validation/v4/is" +) + +type LoginParam struct { + RedirectURL string `form:"redirect_url"` +} + +type OAuthParam struct { + Action string `json:"action,omitempty"` + TenantID string `json:"tenant_id,omitempty"` + Role string `json:"role,omitempty"` + Email string `json:"email,omitempty"` +} + +type InviteParam struct { + Email string `json:"email"` + Role string `json:"role"` +} + +func (v InviteParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Email, validation.Required, is.Email), + validation.Field(&v.Role, validation.Required, validation.In(model.RoleSuperAdmin, model.RoleAdmin, model.RoleUser)), + ) +} + +type ArchivedParam struct { + Archived bool `form:"archived"` +} + +type CreateConnectorParam struct { + Name string `json:"name,omitempty"` + Source string `json:"source,omitempty"` + ConnectorSpecificConfig model.JSONMap `json:"connector_specific_config,omitempty"` + RefreshFreq int `json:"refresh_freq,omitempty"` + Shared bool `json:"shared,omitempty"` + Disabled bool `json:"disabled,omitempty"` +} + +func (v CreateConnectorParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Source, validation.Required, + validation.By(func(value interface{}) error { + if st, ok := model.AllSourceTypes[model.SourceType(v.Source)]; !ok || !st.IsImplemented { + return fmt.Errorf("invalid source type") + } + return nil + }))) +} + +type UpdateConnectorParam struct { + Name string `json:"name,omitempty"` + ConnectorSpecificConfig model.JSONMap `json:"connector_specific_config,omitempty"` + RefreshFreq int `json:"refresh_freq,omitempty"` + Shared bool `json:"shared,omitempty"` + Status string `json:"status"` +} + +func (v UpdateConnectorParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Name, validation.Required), + validation.Field(&v.ConnectorSpecificConfig, validation.Required), + validation.Field(&v.RefreshFreq, validation.Required), + validation.Field(&v.Status, validation.In("", model.ConnectorStatusReadyToProcessed)), + ) +} + +type AddUserParam struct { + Email string `json:"email"` + Role string `json:"role"` +} + +func (v AddUserParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Email, validation.Required, is.Email), + validation.Field(&v.Role, validation.Required, validation.In(model.RoleSuperAdmin, model.RoleUser, model.RoleAdmin)), + ) +} + +type EditUserParam struct { + Role string `json:"role"` +} + +func (v EditUserParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Role, validation.Required, validation.In(model.RoleSuperAdmin, model.RoleUser, model.RoleAdmin)), + ) +} diff --git a/src/backend/core/parameters/persona.go b/src/backend/core/parameters/persona.go new file mode 100644 index 00000000..6d21944f --- /dev/null +++ b/src/backend/core/parameters/persona.go @@ -0,0 +1,42 @@ +package parameters + +import ( + "fmt" + validation "github.com/go-ozzo/ozzo-validation/v4" + "github.com/sashabaranov/go-openai" +) + +var SupportedModels = map[string]bool{ + openai.GPT3Dot5Turbo: true, +} + +type PersonaParam struct { + Name string `json:"name"` + Description string `json:"description"` + ModelID string `json:"model_id"` + URL string `json:"url"` + APIKey string `json:"api_key"` + Endpoint string `json:"endpoint"` + SystemPrompt string `json:"system_prompt"` + TaskPrompt string `json:"task_prompt"` + StarterMessages []*StarterMessage `json:"starter_messages,omitempty"` +} + +type StarterMessage struct { + Name string `json:"name"` + Message string `json:"message"` + Description string `json:"description"` +} + +func (v PersonaParam) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Name, validation.Required), + validation.Field(&v.ModelID, validation.Required, + validation.By(func(value interface{}) error { + if _, ok := SupportedModels[v.ModelID]; !ok { + return fmt.Errorf("model %s not supported", v.ModelID) + } + return nil + })), + validation.Field(&v.APIKey, validation.Required)) +} diff --git a/src/backend/core/proto/client.go b/src/backend/core/proto/client.go new file mode 100644 index 00000000..92256db4 --- /dev/null +++ b/src/backend/core/proto/client.go @@ -0,0 +1 @@ +package proto diff --git a/src/backend/core/proto/connector_messages.pb.go b/src/backend/core/proto/connector_messages.pb.go new file mode 100644 index 00000000..bccfe62c --- /dev/null +++ b/src/backend/core/proto/connector_messages.pb.go @@ -0,0 +1,346 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: connector_messages.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ConnectorRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Params map[string]string `protobuf:"bytes,2,rep,name=params,proto3" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *ConnectorRequest) Reset() { + *x = ConnectorRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnectorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectorRequest) ProtoMessage() {} + +func (x *ConnectorRequest) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectorRequest.ProtoReflect.Descriptor instead. +func (*ConnectorRequest) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{0} +} + +func (x *ConnectorRequest) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *ConnectorRequest) GetParams() map[string]string { + if x != nil { + return x.Params + } + return nil +} + +type EmbeddAsyncRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + ChunkId int64 `protobuf:"varint,2,opt,name=chunk_id,json=chunkId,proto3" json:"chunk_id,omitempty"` + Content string `protobuf:"bytes,3,opt,name=content,proto3" json:"content,omitempty"` + ModelId string `protobuf:"bytes,4,opt,name=model_id,json=modelId,proto3" json:"model_id,omitempty"` +} + +func (x *EmbeddAsyncRequest) Reset() { + *x = EmbeddAsyncRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddAsyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddAsyncRequest) ProtoMessage() {} + +func (x *EmbeddAsyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddAsyncRequest.ProtoReflect.Descriptor instead. +func (*EmbeddAsyncRequest) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{1} +} + +func (x *EmbeddAsyncRequest) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *EmbeddAsyncRequest) GetChunkId() int64 { + if x != nil { + return x.ChunkId + } + return 0 +} + +func (x *EmbeddAsyncRequest) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddAsyncRequest) GetModelId() string { + if x != nil { + return x.ModelId + } + return "" +} + +type EmbeddAsyncResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + ChunkId int64 `protobuf:"varint,2,opt,name=chunk_id,json=chunkId,proto3" json:"chunk_id,omitempty"` + Content string `protobuf:"bytes,3,opt,name=content,proto3" json:"content,omitempty"` + Vector []float32 `protobuf:"fixed32,4,rep,packed,name=vector,proto3" json:"vector,omitempty"` +} + +func (x *EmbeddAsyncResponse) Reset() { + *x = EmbeddAsyncResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddAsyncResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddAsyncResponse) ProtoMessage() {} + +func (x *EmbeddAsyncResponse) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddAsyncResponse.ProtoReflect.Descriptor instead. +func (*EmbeddAsyncResponse) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{2} +} + +func (x *EmbeddAsyncResponse) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *EmbeddAsyncResponse) GetChunkId() int64 { + if x != nil { + return x.ChunkId + } + return 0 +} + +func (x *EmbeddAsyncResponse) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddAsyncResponse) GetVector() []float32 { + if x != nil { + return x.Vector + } + return nil +} + +var File_connector_messages_proto protoreflect.FileDescriptor + +var file_connector_messages_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, + 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, 0x22, 0x9f, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x40, 0x0a, 0x06, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, + 0x6d, 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, + 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x85, 0x01, 0x0a, 0x12, 0x45, 0x6d, 0x62, + 0x65, 0x64, 0x64, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, + 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x69, + 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x49, 0x64, + 0x22, 0x83, 0x01, 0x0a, 0x13, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x41, 0x73, 0x79, 0x6e, 0x63, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, + 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, + 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_connector_messages_proto_rawDescOnce sync.Once + file_connector_messages_proto_rawDescData = file_connector_messages_proto_rawDesc +) + +func file_connector_messages_proto_rawDescGZIP() []byte { + file_connector_messages_proto_rawDescOnce.Do(func() { + file_connector_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_connector_messages_proto_rawDescData) + }) + return file_connector_messages_proto_rawDescData +} + +var file_connector_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_connector_messages_proto_goTypes = []interface{}{ + (*ConnectorRequest)(nil), // 0: com.cognix.ConnectorRequest + (*EmbeddAsyncRequest)(nil), // 1: com.cognix.EmbeddAsyncRequest + (*EmbeddAsyncResponse)(nil), // 2: com.cognix.EmbeddAsyncResponse + nil, // 3: com.cognix.ConnectorRequest.ParamsEntry +} +var file_connector_messages_proto_depIdxs = []int32{ + 3, // 0: com.cognix.ConnectorRequest.params:type_name -> com.cognix.ConnectorRequest.ParamsEntry + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_connector_messages_proto_init() } +func file_connector_messages_proto_init() { + if File_connector_messages_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_connector_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConnectorRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_connector_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddAsyncRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_connector_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddAsyncResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_connector_messages_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_connector_messages_proto_goTypes, + DependencyIndexes: file_connector_messages_proto_depIdxs, + MessageInfos: file_connector_messages_proto_msgTypes, + }.Build() + File_connector_messages_proto = out.File + file_connector_messages_proto_rawDesc = nil + file_connector_messages_proto_goTypes = nil + file_connector_messages_proto_depIdxs = nil +} diff --git a/src/backend/core/proto/embed_service.pb.go b/src/backend/core/proto/embed_service.pb.go new file mode 100644 index 00000000..fc262299 --- /dev/null +++ b/src/backend/core/proto/embed_service.pb.go @@ -0,0 +1,225 @@ +// gRPC Service + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: embed_service.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EmbedRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + Model string `protobuf:"bytes,2,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *EmbedRequest) Reset() { + *x = EmbedRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_embed_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbedRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbedRequest) ProtoMessage() {} + +func (x *EmbedRequest) ProtoReflect() protoreflect.Message { + mi := &file_embed_service_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbedRequest.ProtoReflect.Descriptor instead. +func (*EmbedRequest) Descriptor() ([]byte, []int) { + return file_embed_service_proto_rawDescGZIP(), []int{0} +} + +func (x *EmbedRequest) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbedRequest) GetModel() string { + if x != nil { + return x.Model + } + return "" +} + +type EmbedResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Vector []float32 `protobuf:"fixed32,1,rep,packed,name=vector,proto3" json:"vector,omitempty"` +} + +func (x *EmbedResponse) Reset() { + *x = EmbedResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_embed_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbedResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbedResponse) ProtoMessage() {} + +func (x *EmbedResponse) ProtoReflect() protoreflect.Message { + mi := &file_embed_service_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbedResponse.ProtoReflect.Descriptor instead. +func (*EmbedResponse) Descriptor() ([]byte, []int) { + return file_embed_service_proto_rawDescGZIP(), []int{1} +} + +func (x *EmbedResponse) GetVector() []float32 { + if x != nil { + return x.Vector + } + return nil +} + +var File_embed_service_proto protoreflect.FileDescriptor + +var file_embed_service_proto_rawDesc = []byte{ + 0x0a, 0x13, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, + 0x78, 0x22, 0x3e, 0x0a, 0x0c, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x22, 0x27, 0x0a, 0x0d, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x32, 0x54, 0x0a, 0x0c, 0x45, 0x6d, + 0x62, 0x65, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x47, 0x65, + 0x74, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x18, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, + 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, + 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_embed_service_proto_rawDescOnce sync.Once + file_embed_service_proto_rawDescData = file_embed_service_proto_rawDesc +) + +func file_embed_service_proto_rawDescGZIP() []byte { + file_embed_service_proto_rawDescOnce.Do(func() { + file_embed_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_embed_service_proto_rawDescData) + }) + return file_embed_service_proto_rawDescData +} + +var file_embed_service_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_embed_service_proto_goTypes = []interface{}{ + (*EmbedRequest)(nil), // 0: com.cognix.EmbedRequest + (*EmbedResponse)(nil), // 1: com.cognix.EmbedResponse +} +var file_embed_service_proto_depIdxs = []int32{ + 0, // 0: com.cognix.EmbedService.GetEmbeding:input_type -> com.cognix.EmbedRequest + 1, // 1: com.cognix.EmbedService.GetEmbeding:output_type -> com.cognix.EmbedResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_embed_service_proto_init() } +func file_embed_service_proto_init() { + if File_embed_service_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_embed_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbedRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_embed_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbedResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_embed_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_embed_service_proto_goTypes, + DependencyIndexes: file_embed_service_proto_depIdxs, + MessageInfos: file_embed_service_proto_msgTypes, + }.Build() + File_embed_service_proto = out.File + file_embed_service_proto_rawDesc = nil + file_embed_service_proto_goTypes = nil + file_embed_service_proto_depIdxs = nil +} diff --git a/src/backend/core/proto/embed_service_grpc.pb.go b/src/backend/core/proto/embed_service_grpc.pb.go new file mode 100644 index 00000000..d636a923 --- /dev/null +++ b/src/backend/core/proto/embed_service_grpc.pb.go @@ -0,0 +1,105 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.12.4 +// source: embed_service.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// EmbedServiceClient is the client API for EmbedService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type EmbedServiceClient interface { + GetEmbeding(ctx context.Context, in *EmbedRequest, opts ...grpc.CallOption) (*EmbedResponse, error) +} + +type embedServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewEmbedServiceClient(cc grpc.ClientConnInterface) EmbedServiceClient { + return &embedServiceClient{cc} +} + +func (c *embedServiceClient) GetEmbeding(ctx context.Context, in *EmbedRequest, opts ...grpc.CallOption) (*EmbedResponse, error) { + out := new(EmbedResponse) + err := c.cc.Invoke(ctx, "/com.cognix.EmbedService/GetEmbeding", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// EmbedServiceServer is the server API for EmbedService service. +// All implementations must embed UnimplementedEmbedServiceServer +// for forward compatibility +type EmbedServiceServer interface { + GetEmbeding(context.Context, *EmbedRequest) (*EmbedResponse, error) + mustEmbedUnimplementedEmbedServiceServer() +} + +// UnimplementedEmbedServiceServer must be embedded to have forward compatible implementations. +type UnimplementedEmbedServiceServer struct { +} + +func (UnimplementedEmbedServiceServer) GetEmbeding(context.Context, *EmbedRequest) (*EmbedResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetEmbeding not implemented") +} +func (UnimplementedEmbedServiceServer) mustEmbedUnimplementedEmbedServiceServer() {} + +// UnsafeEmbedServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EmbedServiceServer will +// result in compilation errors. +type UnsafeEmbedServiceServer interface { + mustEmbedUnimplementedEmbedServiceServer() +} + +func RegisterEmbedServiceServer(s grpc.ServiceRegistrar, srv EmbedServiceServer) { + s.RegisterService(&EmbedService_ServiceDesc, srv) +} + +func _EmbedService_GetEmbeding_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EmbedRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EmbedServiceServer).GetEmbeding(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/com.cognix.EmbedService/GetEmbeding", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EmbedServiceServer).GetEmbeding(ctx, req.(*EmbedRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// EmbedService_ServiceDesc is the grpc.ServiceDesc for EmbedService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var EmbedService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "com.cognix.EmbedService", + HandlerType: (*EmbedServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetEmbeding", + Handler: _EmbedService_GetEmbeding_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "embed_service.proto", +} diff --git a/src/backend/core/proto/file_type.pb.go b/src/backend/core/proto/file_type.pb.go new file mode 100644 index 00000000..c2d3c691 --- /dev/null +++ b/src/backend/core/proto/file_type.pb.go @@ -0,0 +1,217 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: file_type.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileType int32 + +const ( + // ORCHESTRATOR SEND TO CONNECTOR: + FileType_UNKNOWN FileType = 0 + FileType_PDF FileType = 1 + FileType_XPS FileType = 2 + FileType_TXT FileType = 3 + FileType_DOCX FileType = 4 + FileType_DOC FileType = 5 + FileType_XLSX FileType = 6 + FileType_XLS FileType = 7 + FileType_PPTX FileType = 8 + FileType_PPT FileType = 9 + FileType_HWPX FileType = 10 + FileType_EPUB FileType = 11 + FileType_MOBI FileType = 12 + FileType_FB2 FileType = 13 + FileType_CBZ FileType = 14 + FileType_MD FileType = 15 + // ORCHESTRATOR SEND TO SEMANTIC + FileType_URL FileType = 16 + FileType_YT FileType = 17 + // ORCHESTRATOR SEND TO WHISPER + FileType_MP4 FileType = 18 + FileType_MP3 FileType = 19 + FileType_MPEG FileType = 20 + FileType_MPGA FileType = 21 + FileType_M4A FileType = 22 + FileType_WAV FileType = 23 + FileType_WEBM FileType = 24 + FileType_MOV FileType = 25 +) + +// Enum value maps for FileType. +var ( + FileType_name = map[int32]string{ + 0: "UNKNOWN", + 1: "PDF", + 2: "XPS", + 3: "TXT", + 4: "DOCX", + 5: "DOC", + 6: "XLSX", + 7: "XLS", + 8: "PPTX", + 9: "PPT", + 10: "HWPX", + 11: "EPUB", + 12: "MOBI", + 13: "FB2", + 14: "CBZ", + 15: "MD", + 16: "URL", + 17: "YT", + 18: "MP4", + 19: "MP3", + 20: "MPEG", + 21: "MPGA", + 22: "M4A", + 23: "WAV", + 24: "WEBM", + 25: "MOV", + } + FileType_value = map[string]int32{ + "UNKNOWN": 0, + "PDF": 1, + "XPS": 2, + "TXT": 3, + "DOCX": 4, + "DOC": 5, + "XLSX": 6, + "XLS": 7, + "PPTX": 8, + "PPT": 9, + "HWPX": 10, + "EPUB": 11, + "MOBI": 12, + "FB2": 13, + "CBZ": 14, + "MD": 15, + "URL": 16, + "YT": 17, + "MP4": 18, + "MP3": 19, + "MPEG": 20, + "MPGA": 21, + "M4A": 22, + "WAV": 23, + "WEBM": 24, + "MOV": 25, + } +) + +func (x FileType) Enum() *FileType { + p := new(FileType) + *p = x + return p +} + +func (x FileType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FileType) Descriptor() protoreflect.EnumDescriptor { + return file_file_type_proto_enumTypes[0].Descriptor() +} + +func (FileType) Type() protoreflect.EnumType { + return &file_file_type_proto_enumTypes[0] +} + +func (x FileType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FileType.Descriptor instead. +func (FileType) EnumDescriptor() ([]byte, []int) { + return file_file_type_proto_rawDescGZIP(), []int{0} +} + +var File_file_type_proto protoreflect.FileDescriptor + +var file_file_type_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, 0x2a, 0xff, 0x01, + 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x44, 0x46, 0x10, 0x01, + 0x12, 0x07, 0x0a, 0x03, 0x58, 0x50, 0x53, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x58, 0x54, + 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x43, 0x58, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, + 0x44, 0x4f, 0x43, 0x10, 0x05, 0x12, 0x08, 0x0a, 0x04, 0x58, 0x4c, 0x53, 0x58, 0x10, 0x06, 0x12, + 0x07, 0x0a, 0x03, 0x58, 0x4c, 0x53, 0x10, 0x07, 0x12, 0x08, 0x0a, 0x04, 0x50, 0x50, 0x54, 0x58, + 0x10, 0x08, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x50, 0x54, 0x10, 0x09, 0x12, 0x08, 0x0a, 0x04, 0x48, + 0x57, 0x50, 0x58, 0x10, 0x0a, 0x12, 0x08, 0x0a, 0x04, 0x45, 0x50, 0x55, 0x42, 0x10, 0x0b, 0x12, + 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x42, 0x49, 0x10, 0x0c, 0x12, 0x07, 0x0a, 0x03, 0x46, 0x42, 0x32, + 0x10, 0x0d, 0x12, 0x07, 0x0a, 0x03, 0x43, 0x42, 0x5a, 0x10, 0x0e, 0x12, 0x06, 0x0a, 0x02, 0x4d, + 0x44, 0x10, 0x0f, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x10, 0x10, 0x12, 0x06, 0x0a, 0x02, + 0x59, 0x54, 0x10, 0x11, 0x12, 0x07, 0x0a, 0x03, 0x4d, 0x50, 0x34, 0x10, 0x12, 0x12, 0x07, 0x0a, + 0x03, 0x4d, 0x50, 0x33, 0x10, 0x13, 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x50, 0x45, 0x47, 0x10, 0x14, + 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x50, 0x47, 0x41, 0x10, 0x15, 0x12, 0x07, 0x0a, 0x03, 0x4d, 0x34, + 0x41, 0x10, 0x16, 0x12, 0x07, 0x0a, 0x03, 0x57, 0x41, 0x56, 0x10, 0x17, 0x12, 0x08, 0x0a, 0x04, + 0x57, 0x45, 0x42, 0x4d, 0x10, 0x18, 0x12, 0x07, 0x0a, 0x03, 0x4d, 0x4f, 0x56, 0x10, 0x19, 0x42, + 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_file_type_proto_rawDescOnce sync.Once + file_file_type_proto_rawDescData = file_file_type_proto_rawDesc +) + +func file_file_type_proto_rawDescGZIP() []byte { + file_file_type_proto_rawDescOnce.Do(func() { + file_file_type_proto_rawDescData = protoimpl.X.CompressGZIP(file_file_type_proto_rawDescData) + }) + return file_file_type_proto_rawDescData +} + +var file_file_type_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_file_type_proto_goTypes = []interface{}{ + (FileType)(0), // 0: com.cognix.FileType +} +var file_file_type_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_file_type_proto_init() } +func file_file_type_proto_init() { + if File_file_type_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_file_type_proto_rawDesc, + NumEnums: 1, + NumMessages: 0, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_file_type_proto_goTypes, + DependencyIndexes: file_file_type_proto_depIdxs, + EnumInfos: file_file_type_proto_enumTypes, + }.Build() + File_file_type_proto = out.File + file_file_type_proto_rawDesc = nil + file_file_type_proto_goTypes = nil + file_file_type_proto_depIdxs = nil +} diff --git a/src/backend/core/proto/semantic_data.pb.go b/src/backend/core/proto/semantic_data.pb.go new file mode 100644 index 00000000..4d702d3f --- /dev/null +++ b/src/backend/core/proto/semantic_data.pb.go @@ -0,0 +1,243 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: semantic_data.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SemanticData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This is the url where the file is located. + // Based on the chunking type it will be a WEB URL (HTML type) + // Will be an S3/MINIO link with a proper authentication in case of a file + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + UrlRecursive bool `protobuf:"varint,2,opt,name=url_recursive,json=urlRecursive,proto3" json:"url_recursive,omitempty"` + SiteMap string `protobuf:"bytes,3,opt,name=site_map,json=siteMap,proto3" json:"site_map,omitempty"` + SearchForSitemap bool `protobuf:"varint,4,opt,name=search_for_sitemap,json=searchForSitemap,proto3" json:"search_for_sitemap,omitempty"` + DocumentId int64 `protobuf:"varint,5,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + ConnectorId int64 `protobuf:"varint,6,opt,name=connector_id,json=connectorId,proto3" json:"connector_id,omitempty"` + FileType FileType `protobuf:"varint,7,opt,name=file_type,json=fileType,proto3,enum=com.cognix.FileType" json:"file_type,omitempty"` + CollectionName string `protobuf:"bytes,8,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"` + ModelName string `protobuf:"bytes,9,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"` + ModelDimension int32 `protobuf:"varint,10,opt,name=model_dimension,json=modelDimension,proto3" json:"model_dimension,omitempty"` +} + +func (x *SemanticData) Reset() { + *x = SemanticData{} + if protoimpl.UnsafeEnabled { + mi := &file_semantic_data_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SemanticData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SemanticData) ProtoMessage() {} + +func (x *SemanticData) ProtoReflect() protoreflect.Message { + mi := &file_semantic_data_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SemanticData.ProtoReflect.Descriptor instead. +func (*SemanticData) Descriptor() ([]byte, []int) { + return file_semantic_data_proto_rawDescGZIP(), []int{0} +} + +func (x *SemanticData) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *SemanticData) GetUrlRecursive() bool { + if x != nil { + return x.UrlRecursive + } + return false +} + +func (x *SemanticData) GetSiteMap() string { + if x != nil { + return x.SiteMap + } + return "" +} + +func (x *SemanticData) GetSearchForSitemap() bool { + if x != nil { + return x.SearchForSitemap + } + return false +} + +func (x *SemanticData) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *SemanticData) GetConnectorId() int64 { + if x != nil { + return x.ConnectorId + } + return 0 +} + +func (x *SemanticData) GetFileType() FileType { + if x != nil { + return x.FileType + } + return FileType_UNKNOWN +} + +func (x *SemanticData) GetCollectionName() string { + if x != nil { + return x.CollectionName + } + return "" +} + +func (x *SemanticData) GetModelName() string { + if x != nil { + return x.ModelName + } + return "" +} + +func (x *SemanticData) GetModelDimension() int32 { + if x != nil { + return x.ModelDimension + } + return 0 +} + +var File_semantic_data_proto protoreflect.FileDescriptor + +var file_semantic_data_proto_rawDesc = []byte{ + 0x0a, 0x13, 0x73, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, + 0x78, 0x1a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0xf6, 0x02, 0x0a, 0x0c, 0x53, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x44, + 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x75, 0x72, 0x6c, 0x5f, 0x72, 0x65, 0x63, + 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x75, 0x72, + 0x6c, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, + 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, + 0x74, 0x65, 0x4d, 0x61, 0x70, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x5f, + 0x66, 0x6f, 0x72, 0x5f, 0x73, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x10, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x46, 0x6f, 0x72, 0x53, 0x69, 0x74, 0x65, + 0x6d, 0x61, 0x70, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6d, + 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x64, 0x69, 0x6d, 0x65, + 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x44, 0x69, 0x6d, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x1a, 0x5a, 0x18, 0x62, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_semantic_data_proto_rawDescOnce sync.Once + file_semantic_data_proto_rawDescData = file_semantic_data_proto_rawDesc +) + +func file_semantic_data_proto_rawDescGZIP() []byte { + file_semantic_data_proto_rawDescOnce.Do(func() { + file_semantic_data_proto_rawDescData = protoimpl.X.CompressGZIP(file_semantic_data_proto_rawDescData) + }) + return file_semantic_data_proto_rawDescData +} + +var file_semantic_data_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_semantic_data_proto_goTypes = []interface{}{ + (*SemanticData)(nil), // 0: com.cognix.SemanticData + (FileType)(0), // 1: com.cognix.FileType +} +var file_semantic_data_proto_depIdxs = []int32{ + 1, // 0: com.cognix.SemanticData.file_type:type_name -> com.cognix.FileType + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_semantic_data_proto_init() } +func file_semantic_data_proto_init() { + if File_semantic_data_proto != nil { + return + } + file_file_type_proto_init() + if !protoimpl.UnsafeEnabled { + file_semantic_data_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SemanticData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_semantic_data_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_semantic_data_proto_goTypes, + DependencyIndexes: file_semantic_data_proto_depIdxs, + MessageInfos: file_semantic_data_proto_msgTypes, + }.Build() + File_semantic_data_proto = out.File + file_semantic_data_proto_rawDesc = nil + file_semantic_data_proto_goTypes = nil + file_semantic_data_proto_depIdxs = nil +} diff --git a/src/backend/core/proto/whisper_data.pb.go b/src/backend/core/proto/whisper_data.pb.go new file mode 100644 index 00000000..dae9cc04 --- /dev/null +++ b/src/backend/core/proto/whisper_data.pb.go @@ -0,0 +1,212 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: whisper_data.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type WhisperData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This is the url where the file is located. + // Based on the chunking type it will be a WEB URL (HTML type) + // Will be an S3/MINIO link with a proper authentication in case of a file + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + DocumentId int64 `protobuf:"varint,2,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + ConnectorId int64 `protobuf:"varint,3,opt,name=connector_id,json=connectorId,proto3" json:"connector_id,omitempty"` + FileType FileType `protobuf:"varint,4,opt,name=file_type,json=fileType,proto3,enum=com.cognix.FileType" json:"file_type,omitempty"` + CollectionName string `protobuf:"bytes,5,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"` + ModelName string `protobuf:"bytes,6,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"` + ModelDimension int32 `protobuf:"varint,7,opt,name=model_dimension,json=modelDimension,proto3" json:"model_dimension,omitempty"` +} + +func (x *WhisperData) Reset() { + *x = WhisperData{} + if protoimpl.UnsafeEnabled { + mi := &file_whisper_data_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *WhisperData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WhisperData) ProtoMessage() {} + +func (x *WhisperData) ProtoReflect() protoreflect.Message { + mi := &file_whisper_data_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WhisperData.ProtoReflect.Descriptor instead. +func (*WhisperData) Descriptor() ([]byte, []int) { + return file_whisper_data_proto_rawDescGZIP(), []int{0} +} + +func (x *WhisperData) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *WhisperData) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *WhisperData) GetConnectorId() int64 { + if x != nil { + return x.ConnectorId + } + return 0 +} + +func (x *WhisperData) GetFileType() FileType { + if x != nil { + return x.FileType + } + return FileType_UNKNOWN +} + +func (x *WhisperData) GetCollectionName() string { + if x != nil { + return x.CollectionName + } + return "" +} + +func (x *WhisperData) GetModelName() string { + if x != nil { + return x.ModelName + } + return "" +} + +func (x *WhisperData) GetModelDimension() int32 { + if x != nil { + return x.ModelDimension + } + return 0 +} + +var File_whisper_data_proto protoreflect.FileDescriptor + +var file_whisper_data_proto_rawDesc = []byte{ + 0x0a, 0x12, 0x77, 0x68, 0x69, 0x73, 0x70, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, + 0x1a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x87, 0x02, 0x0a, 0x0b, 0x57, 0x68, 0x69, 0x73, 0x70, 0x65, 0x72, 0x44, 0x61, 0x74, + 0x61, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x75, 0x72, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6d, + 0x2e, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x78, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x64, 0x69, 0x6d, 0x65, + 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x44, 0x69, 0x6d, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x1a, 0x5a, 0x18, 0x62, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_whisper_data_proto_rawDescOnce sync.Once + file_whisper_data_proto_rawDescData = file_whisper_data_proto_rawDesc +) + +func file_whisper_data_proto_rawDescGZIP() []byte { + file_whisper_data_proto_rawDescOnce.Do(func() { + file_whisper_data_proto_rawDescData = protoimpl.X.CompressGZIP(file_whisper_data_proto_rawDescData) + }) + return file_whisper_data_proto_rawDescData +} + +var file_whisper_data_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_whisper_data_proto_goTypes = []interface{}{ + (*WhisperData)(nil), // 0: com.cognix.WhisperData + (FileType)(0), // 1: com.cognix.FileType +} +var file_whisper_data_proto_depIdxs = []int32{ + 1, // 0: com.cognix.WhisperData.file_type:type_name -> com.cognix.FileType + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_whisper_data_proto_init() } +func file_whisper_data_proto_init() { + if File_whisper_data_proto != nil { + return + } + file_file_type_proto_init() + if !protoimpl.UnsafeEnabled { + file_whisper_data_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*WhisperData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_whisper_data_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_whisper_data_proto_goTypes, + DependencyIndexes: file_whisper_data_proto_depIdxs, + MessageInfos: file_whisper_data_proto_msgTypes, + }.Build() + File_whisper_data_proto = out.File + file_whisper_data_proto_rawDesc = nil + file_whisper_data_proto_goTypes = nil + file_whisper_data_proto_depIdxs = nil +} diff --git a/src/backend/core/repository/chat.go b/src/backend/core/repository/chat.go new file mode 100644 index 00000000..fa21c22d --- /dev/null +++ b/src/backend/core/repository/chat.go @@ -0,0 +1,114 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/go-pg/pg/v10/orm" + "github.com/google/uuid" +) + +type ChatRepository interface { + GetSessions(ctx context.Context, userID uuid.UUID) ([]*model.ChatSession, error) + GetSessionByID(ctx context.Context, userID uuid.UUID, id int64) (*model.ChatSession, error) + CreateSession(ctx context.Context, session *model.ChatSession) error + SendMessage(ctx context.Context, message *model.ChatMessage) error + UpdateMessage(ctx context.Context, message *model.ChatMessage) error + GetMessageByIDAndUserID(ctx context.Context, id int64, userID uuid.UUID) (*model.ChatMessage, error) + MessageFeedback(ctx context.Context, feedback *model.ChatMessageFeedback) error +} + +type chatRepository struct { + db *pg.DB +} + +func (r *chatRepository) GetMessageByIDAndUserID(ctx context.Context, id int64, userID uuid.UUID) (*model.ChatMessage, error) { + var message model.ChatMessage + if err := r.db.Model(&message). + Relation("Feedback"). + Join("inner join chat_sessions on chat_sessions.id = chat_message.chat_session_id and chat_sessions.user_id = ?", userID). + Where("chat_message.id = ?", id).First(); err != nil { + return nil, utils.NotFound.Wrap(err, "cannot find message by id") + } + return &message, nil +} + +func (r *chatRepository) MessageFeedback(ctx context.Context, feedback *model.ChatMessageFeedback) error { + stm := r.db.WithContext(ctx).Model(feedback) + if feedback.ID.IntPart() == 0 { + if _, err := stm.Insert(); err != nil { + return utils.Internal.Wrap(err, "can not add feedback") + } + return nil + } + if _, err := stm.Where("id = ?", feedback.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not update feedback") + } + return nil +} + +func (r *chatRepository) SendMessage(ctx context.Context, message *model.ChatMessage) error { + if _, err := r.db.WithContext(ctx).Model(message).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not save message") + } + return nil +} +func (r *chatRepository) UpdateMessage(ctx context.Context, message *model.ChatMessage) error { + return r.db.RunInTransaction(ctx, func(tx *pg.Tx) error { + if _, err := tx.Model(message).Where("id = ?", message.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not save message") + } + if len(message.DocumentPairs) > 0 { + if _, err := tx.Model(&message.DocumentPairs).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create document pairs") + } + } + return nil + }) + +} + +func NewChatRepository(db *pg.DB) ChatRepository { + return &chatRepository{db: db} +} +func (r *chatRepository) GetSessions(ctx context.Context, userID uuid.UUID) ([]*model.ChatSession, error) { + sessions := make([]*model.ChatSession, 0) + if err := r.db.WithContext(ctx).Model(&sessions). + Where("user_id = ?", userID). + Where("deleted_date is null"). + Order("creation_date desc").Select(); err != nil { + return nil, utils.NotFound.Wrapf(err, "can not find sessions") + } + return sessions, nil +} + +func (r *chatRepository) GetSessionByID(ctx context.Context, userID uuid.UUID, id int64) (*model.ChatSession, error) { + var session model.ChatSession + if err := r.db.WithContext(ctx).Model(&session). + Where("chat_session.user_id = ?", userID). + Where("chat_session.id = ?", id). + Relation("Persona"). + Relation("Persona.Prompt"). + Relation("Persona.LLM"). + Relation("Messages", func(query *orm.Query) (*orm.Query, error) { + return query.Order("time_sent asc"), nil + }). + Relation("Messages.Feedback"). + Relation("Messages.DocumentPairs"). + Relation("Messages.DocumentPairs.Document"). + First(); err != nil { + return nil, utils.NotFound.Wrapf(err, "can not find session") + } + for _, msg := range session.Messages { + _ = msg.AfterSelect(ctx) + } + return &session, nil +} + +func (r *chatRepository) CreateSession(ctx context.Context, session *model.ChatSession) error { + if _, err := r.db.WithContext(ctx).Model(session).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create chat session") + } + return nil +} diff --git a/src/backend/core/repository/client.go b/src/backend/core/repository/client.go new file mode 100644 index 00000000..fe85e56c --- /dev/null +++ b/src/backend/core/repository/client.go @@ -0,0 +1,44 @@ +package repository + +import ( + "context" + "github.com/go-pg/pg/v10" + _ "github.com/lib/pq" + "go.uber.org/zap" +) + +type Config struct { + URL string `env:"COCKROACH_CLIENT_DATABASE_URL_GOLANG"` + DebugMode bool `env:"COCKROACH_CLIENT_DB_DEBUG" envDefault:"false"` +} + +func NewDatabase(cfg *Config) (*pg.DB, error) { + opt, err := pg.ParseURL(cfg.URL) + if err != nil { + return nil, err + } + db := pg.Connect(opt) + if cfg.DebugMode { + db.AddQueryHook(dbLogger{}) + } + if err = db.Ping(context.Background()); err != nil { + return nil, err + } + return db, nil +} + +type dbLogger struct{} + +func (d dbLogger) BeforeQuery(c context.Context, q *pg.QueryEvent) (context.Context, error) { + return c, nil +} + +func (d dbLogger) AfterQuery(c context.Context, q *pg.QueryEvent) error { + if query, err := q.FormattedQuery(); err != nil { + zap.S().Infof("[SQL]: %s", err.Error()) + } else { + zap.S().Infof("[SQL]: %s", string(query)) + } + + return nil +} diff --git a/src/backend/core/repository/connector.go b/src/backend/core/repository/connector.go new file mode 100644 index 00000000..d97b732d --- /dev/null +++ b/src/backend/core/repository/connector.go @@ -0,0 +1,119 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/go-pg/pg/v10/orm" + "github.com/google/uuid" +) + +type ( + ConnectorRepository interface { + GetActive(ctx context.Context) ([]*model.Connector, error) + GetAllByUser(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) + GetByIDAndUser(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) + GetByID(ctx context.Context, id int64) (*model.Connector, error) + GetBySource(ctx context.Context, tenantID, userID uuid.UUID, source model.SourceType) (*model.Connector, error) + Create(ctx context.Context, connector *model.Connector) error + Update(ctx context.Context, connector *model.Connector) error + } + connectorRepository struct { + db *pg.DB + } +) + +func (r *connectorRepository) GetBySource(ctx context.Context, tenantID, userID uuid.UUID, source model.SourceType) (*model.Connector, error) { + var connector model.Connector + if err := r.db.WithContext(ctx).Model(&connector). + Where("source = ?", source). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + return query.WhereOr("user_id = ?", userID). + WhereOr("tenant_id = ?", tenantID), nil + }).First(); err != nil { + return nil, utils.NotFound.Wrap(err, "ca not find connector") + } + return &connector, nil +} + +func NewConnectorRepository(db *pg.DB) ConnectorRepository { + return &connectorRepository{db: db} +} + +func (r *connectorRepository) GetAllByUser(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) { + connectors := make([]*model.Connector, 0) + if err := r.db.WithContext(ctx).Model(&connectors). + Where("deleted_date is null"). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + return query.WhereOr("user_id = ?", userID). + WhereOr("tenant_id = ?", tenantID), nil + }).Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not load connectors") + } + return connectors, nil +} + +func (r *connectorRepository) GetByIDAndUser(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) { + var connector model.Connector + if err := r.db.WithContext(ctx).Model(&connector). + Where("id = ?", id). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + return query.WhereOr("user_id = ?", userID). + WhereOr("tenant_id = ?", tenantID), nil + }).First(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not load connector") + } + return &connector, nil +} + +func (r *connectorRepository) GetByID(ctx context.Context, id int64) (*model.Connector, error) { + var connector model.Connector + if err := r.db.WithContext(ctx).Model(&connector). + Relation("Docs"). + Relation("User.EmbeddingModel"). + Where("connector.id = ?", id). + First(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not load connector") + } + connector.DocsMap = make(map[string]*model.Document) + for _, doc := range connector.Docs { + connector.DocsMap[doc.SourceID] = doc + } + return &connector, nil +} + +func (r *connectorRepository) Create(ctx context.Context, connector *model.Connector) error { + stm := r.db.WithContext(ctx).Model(connector) + if _, err := stm.Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create connector") + } + return nil +} + +func (r *connectorRepository) Update(ctx context.Context, connector *model.Connector) error { + if _, err := r.db.WithContext(ctx).Model(connector).Where("id = ?", connector.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not update connector") + } + return nil +} + +func (r *connectorRepository) GetActive(ctx context.Context) ([]*model.Connector, error) { + connectors := make([]*model.Connector, 0) + //todo ask Gian how to do this + + // load connectors with status that might be resending + enabledStatuses := []string{model.ConnectorStatusReadyToProcessed, model.ConnectorStatusSuccess, model.ConnectorStatusError} + + if err := r.db.WithContext(ctx). + Model(&connectors). + Relation("Docs"). + Relation("User.EmbeddingModel"). + Where("status = any(?)", pg.Array(enabledStatuses)). + Where("connector.deleted_date is null"). + Select(); err != nil { + return nil, utils.Internal.Wrapf(err, "can not load connectors: %s ", err.Error()) + } + return connectors, nil + +} diff --git a/src/backend/core/repository/document.go b/src/backend/core/repository/document.go new file mode 100644 index 00000000..a6038d77 --- /dev/null +++ b/src/backend/core/repository/document.go @@ -0,0 +1,97 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/go-pg/pg/v10/orm" + "github.com/lib/pq" + "time" +) + +type ( + DocumentRepository interface { + FindByConnectorIDAndUser(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) + FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) + FindByID(ctx context.Context, id int64) (*model.Document, error) + Create(ctx context.Context, document *model.Document) error + Update(ctx context.Context, document *model.Document) error + DeleteByIDS(ctx context.Context, ids ...int64) error + } + documentRepository struct { + db *pg.DB + } +) + +func (r *documentRepository) FindByID(ctx context.Context, id int64) (*model.Document, error) { + var doc model.Document + if err := r.db.WithContext(ctx).Model(&doc).Where("id = ?", id).Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "document not found") + } + return &doc, nil +} + +func (r *documentRepository) FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) { + documents := make([]*model.Document, 0) + if err := r.db.WithContext(ctx).Model(&documents). + Join("INNER JOIN connectors c ON c.id = connector_id"). + Where("connector_id = ?", connectorID). + Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find documents ") + } + return documents, nil +} + +func (r *documentRepository) FindByConnectorIDAndUser(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) { + documents := make([]*model.Document, 0) + if err := r.db.WithContext(ctx).Model(&documents). + Join("INNER JOIN connectors c ON c.id = connector_id"). + Where("connector_id = ?", connectorID). + Where("c.tenant_id = ?", user.TenantID). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + return query.WhereOr("c.user_id = ? ", user.ID). + WhereOr("c.shared = ?", true), nil + }).Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find documents ") + } + return documents, nil +} + +func (r *documentRepository) Create(ctx context.Context, document *model.Document) error { + document.CreationDate = time.Now().UTC() + stm := r.db.WithContext(ctx).Model(document) + if !document.ParentID.Valid { + stm = stm.ExcludeColumn("parent_id") + } + + if _, err := stm.Insert(); err != nil { + return utils.Internal.Wrapf(err, "can not insert document [%s]", err.Error()) + } + return nil +} + +func (r *documentRepository) Update(ctx context.Context, document *model.Document) error { + stm := r.db.WithContext(ctx).Model(document).Where("id = ? ", document.ID) + if !document.ParentID.Valid { + stm = stm.ExcludeColumn("parent_id") + } + + document.LastUpdate = pg.NullTime{time.Now().UTC()} + if _, err := stm.Update(); err != nil { + return utils.Internal.Wrapf(err, "can not update document [%s]", err.Error()) + } + return nil +} + +func (r *documentRepository) DeleteByIDS(ctx context.Context, ids ...int64) error { + if _, err := r.db.WithContext(ctx).Model(&model.Document{}). + Where("id = any ?", pq.Array(ids)).Delete(); err != nil { + return utils.Internal.Wrapf(err, "can not delete documents [%s]", err.Error()) + } + return nil +} + +func NewDocumentRepository(db *pg.DB) DocumentRepository { + return &documentRepository{db: db} +} diff --git a/src/backend/core/repository/embedding_model.go b/src/backend/core/repository/embedding_model.go new file mode 100644 index 00000000..87bf933d --- /dev/null +++ b/src/backend/core/repository/embedding_model.go @@ -0,0 +1,92 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" + "time" +) + +type ( + EmbeddingModelRepository interface { + GetAll(ctx context.Context, tenantID uuid.UUID, param *parameters.ArchivedParam) ([]*model.EmbeddingModel, error) + GetByID(ctx context.Context, tenantID uuid.UUID, id int64) (*model.EmbeddingModel, error) + GetDefault(ctx context.Context, tenantID uuid.UUID) (*model.EmbeddingModel, error) + Create(ctx context.Context, em *model.EmbeddingModel) error + Update(ctx context.Context, em *model.EmbeddingModel) error + Delete(ctx context.Context, em *model.EmbeddingModel) error + } + embeddingModelRepository struct { + db *pg.DB + } +) + +func (r *embeddingModelRepository) GetDefault(ctx context.Context, tenantID uuid.UUID) (*model.EmbeddingModel, error) { + var em model.EmbeddingModel + if err := r.db.Model(&em).Where("tenant_id = ?", tenantID). + Where("is_active = true"). + Limit(1). + First(); err != nil { + return nil, utils.NotFound.Wrap(err, "Cannot get default embedding model") + } + return &em, nil +} + +func (r *embeddingModelRepository) GetAll(ctx context.Context, tenantID uuid.UUID, param *parameters.ArchivedParam) ([]*model.EmbeddingModel, error) { + ems := make([]*model.EmbeddingModel, 0) + stm := r.db.WithContext(ctx).Model(&ems).Where("tenant_id = ?", tenantID) + if !param.Archived { + stm = stm.Where("deleted_date is null") + } + + if err := stm.Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find embedding models") + } + return ems, nil +} + +func (r *embeddingModelRepository) GetByID(ctx context.Context, tenantID uuid.UUID, id int64) (*model.EmbeddingModel, error) { + var em model.EmbeddingModel + if err := r.db.WithContext(ctx).Model(&em).Where("id = ?", id). + Where("tenant_id = ?", tenantID). + Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find embedding models") + } + return &em, nil +} + +func (r *embeddingModelRepository) Create(ctx context.Context, em *model.EmbeddingModel) error { + if _, err := r.db.WithContext(ctx).Model(em).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create embedding models") + } + return nil +} + +func (r *embeddingModelRepository) Update(ctx context.Context, em *model.EmbeddingModel) error { + em.LastUpdate = pg.NullTime{time.Now().UTC()} + if _, err := r.db.WithContext(ctx).Model(em). + Where("id = ?", em.ID). + Update(); err != nil { + return utils.Internal.Wrap(err, "can not update embedding models") + } + return nil +} + +func (r *embeddingModelRepository) Delete(ctx context.Context, em *model.EmbeddingModel) error { + em.DeletedDate = pg.NullTime{time.Now().UTC()} + if _, err := r.db.WithContext(ctx).Model(em). + Where("id = ?", em.ID). + Update(); err != nil { + return utils.Internal.Wrap(err, "can not update embedding models") + } + return nil +} + +func NewEmbeddingModelRepository(db *pg.DB) EmbeddingModelRepository { + return &embeddingModelRepository{ + db: db, + } +} diff --git a/src/backend/core/repository/llm.go b/src/backend/core/repository/llm.go new file mode 100644 index 00000000..bd27e50b --- /dev/null +++ b/src/backend/core/repository/llm.go @@ -0,0 +1,42 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" +) + +type ( + LLMRepository interface { + GetAll(ctx context.Context) ([]*model.LLM, error) + GetByUserID(ctx context.Context, userID uuid.UUID) (*model.LLM, error) + } + llmRepository struct { + db *pg.DB + } +) + +func (r *llmRepository) GetByUserID(ctx context.Context, userID uuid.UUID) (*model.LLM, error) { + var llm model.LLM + if err := r.db.WithContext(ctx).Model(&llm). + Join("inner join users on llm.tenant_id = users.tenant_id"). + Where("users.id = ?", userID). + Limit(1).First(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find llm ") + } + return &llm, nil +} + +func NewLLMRepository(db *pg.DB) LLMRepository { + return &llmRepository{db: db} +} + +func (r *llmRepository) GetAll(ctx context.Context) ([]*model.LLM, error) { + llms := make([]*model.LLM, 0) + if err := r.db.WithContext(ctx).Model(&llms).Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find llm") + } + return llms, nil +} diff --git a/src/backend/core/repository/module.go b/src/backend/core/repository/module.go new file mode 100644 index 00000000..22c56e8a --- /dev/null +++ b/src/backend/core/repository/module.go @@ -0,0 +1,29 @@ +package repository + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +var DatabaseModule = fx.Options( + fx.Provide(func() (*Config, error) { + cfg := Config{} + err := utils.ReadConfig(&cfg) + return &cfg, err + }, + NewDatabase, + ), +) + +var RepositoriesModule = fx.Options( + fx.Provide( + NewUserRepository, + NewConnectorRepository, + NewLLMRepository, + NewPersonaRepository, + NewChatRepository, + NewDocumentRepository, + NewEmbeddingModelRepository, + NewTenantRepository, + ), +) diff --git a/src/backend/core/repository/persona.go b/src/backend/core/repository/persona.go new file mode 100644 index 00000000..71e05e5f --- /dev/null +++ b/src/backend/core/repository/persona.go @@ -0,0 +1,123 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" +) + +type ( + PersonaRepository interface { + GetAll(ctx context.Context, tenantID uuid.UUID, archived bool) ([]*model.Persona, error) + GetByID(ctx context.Context, id int64, tenantID uuid.UUID, relations ...string) (*model.Persona, error) + Create(ctx context.Context, persona *model.Persona) error + Update(ctx context.Context, persona *model.Persona) error + Archive(ctx context.Context, persona *model.Persona) error + IsExists(ctx context.Context, id int64, tenantID uuid.UUID) (bool, error) + } + personaRepository struct { + db *pg.DB + } +) + +func (r *personaRepository) IsExists(ctx context.Context, id int64, tenantID uuid.UUID) (bool, error) { + exist, err := r.db.WithContext(ctx).Model(&model.Persona{}). + Where("id = ?", id).Where("tenant_id = ?", tenantID). + Exists() + if err != nil { + return false, utils.NotFound.Wrap(err, "can not find persona") + } + return exist, nil +} + +func NewPersonaRepository(db *pg.DB) PersonaRepository { + return &personaRepository{db: db} +} + +func (r *personaRepository) GetAll(ctx context.Context, tenantID uuid.UUID, archived bool) ([]*model.Persona, error) { + personas := make([]*model.Persona, 0) + stm := r.db.WithContext(ctx).Model(&personas). + Relation("LLM.model_id").Relation("LLM.name").Relation("LLM.id"). + Relation("LLM.endpoint"). + Where("persona.tenant_id = ?", tenantID) + + if !archived { + stm = stm.Where("persona.deleted_date IS NULL") + } + if err := stm.Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "personas not found") + } + return personas, nil +} + +func (r *personaRepository) GetByID(ctx context.Context, id int64, tenantID uuid.UUID, relations ...string) (*model.Persona, error) { + var persona model.Persona + stm := r.db.WithContext(ctx).Model(&persona). + Relation("LLM"). + Relation("Prompt"). + Where("persona.id = ?", id). + Where("persona.tenant_id = ?", tenantID) + for _, relation := range relations { + if relation == "LLM" || relation == "Prompt" { + continue + } + stm = stm.Relation(relation) + } + if err := stm.First(); err != nil { + return nil, utils.NotFound.Wrap(err, "persona not found") + } + return &persona, nil +} + +func (r *personaRepository) Create(ctx context.Context, persona *model.Persona) error { + return r.db.RunInTransaction(ctx, func(tx *pg.Tx) error { + if _, err := tx.Model(persona.LLM).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not insert LLM") + } + persona.LlmID = persona.LLM.ID + if _, err := tx.Model(persona).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not insert persona") + } + persona.Prompt.PersonaID = persona.ID + if _, err := tx.Model(persona.Prompt).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not insert prompt") + } + return nil + }) +} + +func (r *personaRepository) Update(ctx context.Context, persona *model.Persona) error { + return r.db.RunInTransaction(ctx, func(tx *pg.Tx) error { + if _, err := tx.Model(persona.LLM).Where("id = ?", persona.LLM.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not update LLM") + } + persona.LlmID = persona.LLM.ID + if _, err := tx.Model(persona).Where("id = ?", persona.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not update persona") + } + persona.Prompt.PersonaID = persona.ID + if _, err := tx.Model(persona.Prompt).Where("id = ?", persona.Prompt.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not update prompt") + } + return nil + }) +} + +func (r *personaRepository) Archive(ctx context.Context, persona *model.Persona) error { + return r.db.RunInTransaction(ctx, func(tx *pg.Tx) error { + if _, err := tx.Model(&model.ChatSession{}).Where("persona_id = ?", persona.ID). + Set("deleted_date = ?", persona.DeletedDate). + Update(); err != nil { + return utils.Internal.Wrap(err, "can not set deleted_date for chat sessions") + } + if _, err := tx.Model(&model.Persona{}).Where("id = ?", persona.ID). + Set("deleted_date = ?", persona.DeletedDate). + Set("last_update = ?", persona.LastUpdate). + Update(); err != nil { + return utils.Internal.Wrap(err, "can not set deleted_date for chat sessions") + } + return nil + }) +} diff --git a/src/backend/core/repository/tenant.go b/src/backend/core/repository/tenant.go new file mode 100644 index 00000000..7447a877 --- /dev/null +++ b/src/backend/core/repository/tenant.go @@ -0,0 +1,30 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" +) + +type ( + TenantRepository interface { + GetUsers(ctx context.Context, tenantID uuid.UUID) ([]*model.User, error) + } + tenantRepository struct { + db *pg.DB + } +) + +func (r *tenantRepository) GetUsers(ctx context.Context, tenantID uuid.UUID) ([]*model.User, error) { + users := make([]*model.User, 0) + if err := r.db.Model(&users).Where("tenant_id = ?", tenantID).Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "cannot get users") + } + return users, nil +} + +func NewTenantRepository(db *pg.DB) TenantRepository { + return &tenantRepository{db: db} +} diff --git a/src/backend/core/repository/user.go b/src/backend/core/repository/user.go new file mode 100644 index 00000000..49768007 --- /dev/null +++ b/src/backend/core/repository/user.go @@ -0,0 +1,87 @@ +package repository + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/go-pg/pg/v10" + "github.com/google/uuid" +) + +type ( + UserRepository interface { + GetByUserName(c context.Context, username string) (*model.User, error) + GetByIDAndTenantID(c context.Context, id, tenantID uuid.UUID) (*model.User, error) + IsUserExists(c context.Context, username string) (bool, error) + RegisterUser(c context.Context, user *model.User) error + Create(c context.Context, user *model.User) error + Update(c context.Context, user *model.User) error + } + // UserRepository provides database operations with User model + userRepository struct { + db *pg.DB + } +) + +func NewUserRepository(db *pg.DB) UserRepository { + return &userRepository{db: db} +} + +func (u *userRepository) GetByUserName(c context.Context, username string) (*model.User, error) { + var user model.User + if err := u.db.WithContext(c).Model(&user).Where("user_name = ?", username).First(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find user") + } + return &user, nil +} + +func (u *userRepository) GetByIDAndTenantID(c context.Context, id, tenantID uuid.UUID) (*model.User, error) { + var user model.User + if err := u.db.WithContext(c).Model(&user). + Where("id = ?", id). + Where("tenant_id = ?", tenantID). + First(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find user") + } + return &user, nil +} +func (u *userRepository) IsUserExists(c context.Context, username string) (bool, error) { + exists, err := u.db.WithContext(c).Model(&model.User{}).Where("user_name = ?", username).Exists() + if err != nil { + return false, utils.NotFound.Wrap(err, "can not find user") + } + return exists, err +} + +func (u *userRepository) Create(c context.Context, user *model.User) error { + if _, err := u.db.WithContext(c).Model(user).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create user") + } + return nil +} +func (u *userRepository) Update(c context.Context, user *model.User) error { + if _, err := u.db.WithContext(c).Model(user).Where("id = ?", user.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not update user") + } + return nil +} +func (u *userRepository) RegisterUser(c context.Context, user *model.User) error { + return u.db.RunInTransaction(c, func(tx *pg.Tx) error { + tenant := model.Tenant{ + ID: user.TenantID, + Name: user.FirstName, + Configuration: nil, + } + if _, err := tx.Model(&tenant).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create tenant") + } + if _, err := tx.Model(user).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create user") + } + if _, err := tx.Model(user.Defaults.EmbeddingModel).Insert(); err != nil { + return utils.Internal.Wrap(err, "can not create default embedding model") + } + + return nil + }) +} diff --git a/src/backend/core/responder/ai.go b/src/backend/core/responder/ai.go new file mode 100644 index 00000000..b69802c1 --- /dev/null +++ b/src/backend/core/responder/ai.go @@ -0,0 +1,112 @@ +package responder + +import ( + "cognix.ch/api/v2/core/ai" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "context" + "github.com/google/uuid" + "strings" + "sync" + "time" +) + +type aiResponder struct { + aiClient ai.OpenAIClient + charRepo repository.ChatRepository + embedding *embedding +} + +func (r *aiResponder) Send(ctx context.Context, + ch chan *Response, + wg *sync.WaitGroup, + user *model.User, noLLM bool, parentMessage *model.ChatMessage, persona *model.Persona) { + defer wg.Done() + message := model.ChatMessage{ + ChatSessionID: parentMessage.ChatSessionID, + ParentMessageID: parentMessage.ID, + MessageType: model.MessageTypeAssistant, + TimeSent: time.Now().UTC(), + ParentMessage: parentMessage, + Message: "You are using Cognix without an LLM. I can give you the documents retrieved in my knowledge. ", + } + if err := r.charRepo.SendMessage(ctx, &message); err != nil { + ch <- &Response{ + IsValid: err == nil, + Type: ResponseMessage, + Message: &message, + } + return + } + + docs, err := r.embedding.FindDocuments(ctx, ch, &message, model.CollectionName(user.ID, uuid.NullUUID{Valid: true, UUID: user.TenantID}), + model.CollectionName(user.ID, uuid.NullUUID{Valid: false})) + if err != nil { + + } + if noLLM { + return + } + messageParts := []string{ + persona.Prompt.SystemPrompt, + parentMessage.Message, + persona.Prompt.TaskPrompt, + } + + for _, doc := range docs { + messageParts = append(messageParts, doc.Content) + if doc.ID.IntPart() != 0 { + message.DocumentPairs = append(message.DocumentPairs, &model.ChatMessageDocumentPair{ + ChatMessageID: message.ID, + DocumentID: doc.ID, + }) + } + } + message.Citations = docs + message.Message = "" + //_ = docs + // docs.Content + // user chat + // system_prompt + // task_prompt + // default_prompt + // llm message format : system prompt \n user chat \n task_prompt \n document content1 \n ...\n document content n ( top 5) + // + response, err := r.aiClient.Request(ctx, strings.Join(messageParts, "\n")) + + if err != nil { + message.Error = err.Error() + } else { + message.Message = response.Message + } + + if errr := r.charRepo.UpdateMessage(ctx, &message); errr != nil { + err = errr + message.Error = err.Error() + } + payload := &Response{ + IsValid: err == nil, + Type: ResponseMessage, + Message: &message, + } + if err != nil { + payload.Type = ResponseError + } + ch <- payload +} + +func NewAIResponder( + aiClient ai.OpenAIClient, + charRepo repository.ChatRepository, + embeddProto proto.EmbedServiceClient, + milvusClinet storage.MilvusClient, + docRepo repository.DocumentRepository, + embeddingModel string, +) ChatResponder { + return &aiResponder{aiClient: aiClient, + charRepo: charRepo, + embedding: NewEmbeddingResponder(embeddProto, milvusClinet, docRepo, embeddingModel), + } +} diff --git a/src/backend/core/responder/embedding.go b/src/backend/core/responder/embedding.go new file mode 100644 index 00000000..cb01eb2b --- /dev/null +++ b/src/backend/core/responder/embedding.go @@ -0,0 +1,116 @@ +package responder + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "context" + "fmt" + "github.com/shopspring/decimal" + "go.uber.org/zap" + "sync" + "time" +) + +type embedding struct { + embedding proto.EmbedServiceClient + milvusClinet storage.MilvusClient + docRepo repository.DocumentRepository + embeddingModel string +} + +func (r *embedding) Send(ctx context.Context, + ch chan *Response, wg *sync.WaitGroup, + user *model.User, + parentMessage *model.ChatMessage, + persona *model.Persona) { + + for i := 0; i < 4; i++ { + ch <- &Response{ + IsValid: true, + Type: ResponseDocument, + Message: nil, + Document: &model.DocumentResponse{ + ID: decimal.NewFromInt(int64(i)), + DocumentID: "11", + Link: fmt.Sprintf("link for document %d", i), + Content: fmt.Sprintf("content of document %d", i), + UpdatedDate: time.Now().UTC().Add(-48 * time.Hour), + MessageID: parentMessage.ID, + }, + } + } + wg.Done() +} + +func (r *embedding) FindDocuments(ctx context.Context, + ch chan *Response, + message *model.ChatMessage, + collectionNames ...string) ([]*model.DocumentResponse, error) { + response, err := r.embedding.GetEmbeding(ctx, &proto.EmbedRequest{ + Content: message.ParentMessage.Message, + Model: r.embeddingModel, + }) + if err != nil { + zap.S().Errorf("embeding service %s ", err.Error()) + ch <- &Response{ + IsValid: false, + Type: ResponseError, + Err: err, + } + return nil, err + } + var result []*model.DocumentResponse + mapResult := make(map[string]*model.DocumentResponse) + for _, collectionName := range collectionNames { + docs, err := r.milvusClinet.Load(ctx, collectionName, response.GetVector()) + if err != nil { + zap.S().Errorf("error loading document from vector database :%s", err.Error()) + continue + } + for _, doc := range docs { + resDocument := &model.DocumentResponse{ + ID: decimal.NewFromInt(doc.DocumentID), + MessageID: message.ID, + Content: doc.Content, + } + if dbDoc, err := r.docRepo.FindByID(ctx, doc.DocumentID); err == nil { + resDocument.Link = dbDoc.OriginalURL + if resDocument.Link == "" { + resDocument.Link = dbDoc.URL + } + resDocument.DocumentID = dbDoc.SourceID + if !dbDoc.LastUpdate.IsZero() { + resDocument.UpdatedDate = dbDoc.LastUpdate.Time + } else { + resDocument.UpdatedDate = dbDoc.CreationDate + } + } + if _, ok := mapResult[resDocument.DocumentID]; ok { + continue + } + mapResult[resDocument.DocumentID] = resDocument + result = append(result, resDocument) + ch <- &Response{ + IsValid: true, + Type: ResponseDocument, + Document: resDocument, + } + } + } + return result, nil +} + +// NewEmbeddingResponder creates new instance os embedder responder +func NewEmbeddingResponder(embeddProto proto.EmbedServiceClient, + milvusClinet storage.MilvusClient, + docRepo repository.DocumentRepository, + embeddingModel string) *embedding { + return &embedding{ + embedding: embeddProto, + milvusClinet: milvusClinet, + embeddingModel: embeddingModel, + docRepo: docRepo, + } +} diff --git a/src/backend/core/responder/manager.go b/src/backend/core/responder/manager.go new file mode 100644 index 00000000..59ddc518 --- /dev/null +++ b/src/backend/core/responder/manager.go @@ -0,0 +1,44 @@ +package responder + +import ( + "cognix.ch/api/v2/core/model" + "context" + "sync" +) + +type Manager struct { + ch chan *Response + wg *sync.WaitGroup + responders []ChatResponder +} + +func (m *Manager) Send(cx context.Context, + user *model.User, + noLLM bool, + parentMessage *model.ChatMessage, + persona *model.Persona) { + for _, responder := range m.responders { + m.wg.Add(1) + go responder.Send(cx, m.ch, m.wg, user, noLLM, parentMessage, persona) + } + m.wg.Wait() + close(m.ch) +} + +func (m *Manager) Receive() (*Response, bool) { + for response := range m.ch { + return response, true + } + return &Response{ + IsValid: true, + Type: ResponseEnd, + }, false +} + +func NewManager(responders ...ChatResponder) *Manager { + return &Manager{ + ch: make(chan *Response, 1), + wg: &sync.WaitGroup{}, + responders: append([]ChatResponder{}, responders...), + } +} diff --git a/src/backend/core/responder/responder.go b/src/backend/core/responder/responder.go new file mode 100644 index 00000000..5365aa24 --- /dev/null +++ b/src/backend/core/responder/responder.go @@ -0,0 +1,26 @@ +package responder + +import ( + "cognix.ch/api/v2/core/model" + "context" + "sync" +) + +const ( + ResponseMessage = "message" + ResponseError = "error" + ResponseDocument = "document" + ResponseEnd = "end" +) + +type Response struct { + IsValid bool + Type string + Message *model.ChatMessage + Document *model.DocumentResponse + Err error +} + +type ChatResponder interface { + Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, noLLM bool, parentMessage *model.ChatMessage, persona *model.Persona) +} diff --git a/src/backend/core/security/jwt.go b/src/backend/core/security/jwt.go new file mode 100644 index 00000000..b46bc41b --- /dev/null +++ b/src/backend/core/security/jwt.go @@ -0,0 +1,61 @@ +package security + +import ( + "cognix.ch/api/v2/core/model" + "fmt" + "github.com/golang-jwt/jwt" + "time" +) + +type ( + Identity struct { + jwt.StandardClaims + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + User *model.User `json:"user"` + } + + JWTService interface { + Create(claim *Identity) (string, error) + ParseAndValidate(string) (*Identity, error) + Refresh(refreshToken string) (string, error) + } + jwtService struct { + jwtSecret string `json:"jwt_secret"` + jwtExpiredTime int `json:"jwt_expired_time"` + } +) + +func NewJWTService(jwtSecret string, jwtExpiredTime int) JWTService { + return &jwtService{jwtSecret: jwtSecret, + jwtExpiredTime: jwtExpiredTime * int(time.Minute)} +} + +func (j *jwtService) Create(identity *Identity) (string, error) { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, identity) + //identity.ExpiresAt = time.Now().Add(time.Duration(j.jwtExpiredTime)).Unix() + tokenString, err := token.SignedString([]byte(j.jwtSecret)) + if err != nil { + return "", err + } + + return tokenString, nil +} + +func (j *jwtService) ParseAndValidate(tokenString string) (*Identity, error) { + var identity Identity + token, err := jwt.ParseWithClaims(tokenString, &identity, func(token *jwt.Token) (interface{}, error) { + return []byte(j.jwtSecret), nil + }) + if err != nil { + return nil, err + } + if !token.Valid { + return nil, fmt.Errorf("invalid token") + } + + return &identity, nil +} +func (j *jwtService) Refresh(refreshToken string) (string, error) { + return refreshToken, nil +} diff --git a/src/backend/core/security/jwt_test.go b/src/backend/core/security/jwt_test.go new file mode 100644 index 00000000..38a89c34 --- /dev/null +++ b/src/backend/core/security/jwt_test.go @@ -0,0 +1,31 @@ +package security + +import ( + "cognix.ch/api/v2/core/model" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestJwtService_Create(t *testing.T) { + + js := NewJWTService("secret", 60) + + identity := Identity{ + AccessToken: "ya29.a0Ad52N3-ivglS4hoLsoLmgGC6Xp-culrkTZEY5sCs1AknfFsxOOiiVIKtEU7zoA7hDUAJ8y2wbaaoRjI7e8h0l_1iOo0cJSLJBId7G7P6nX370UWx_dvMcaS_ExEBG5N64f6ZWNAMXPyoeESkUi7rOzxWf3yqSQlmLgaCgYKAckSARMSFQHGX2Mi5FKu4kDXffW-8Da4is1WnA0169", + RefreshToken: "", + User: &model.User{ + ID: uuid.New(), + TenantID: uuid.New(), + UserName: "andrey.paladiychuk@gmail.com", + FirstName: "Andrii", + LastName: "Paladiichuk", + ExternalID: "102087316285176952515", + Roles: nil, + Tenant: nil, + }, + } + token, err := js.Create(&identity) + assert.NoError(t, err) + t.Log(token) +} diff --git a/src/backend/core/server/auth_middleware.go b/src/backend/core/server/auth_middleware.go new file mode 100644 index 00000000..0605b694 --- /dev/null +++ b/src/backend/core/server/auth_middleware.go @@ -0,0 +1,73 @@ +package server + +import ( + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/utils" + "context" + "github.com/gin-gonic/gin" + "strings" + "time" +) + +const ContextParamUser = "CONTEXT_USER" + +type AuthMiddleware struct { + jwtService security.JWTService + userRepo repository.UserRepository +} + +func NewAuthMiddleware(jwtService security.JWTService, + userRepo repository.UserRepository) *AuthMiddleware { + return &AuthMiddleware{jwtService: jwtService, + userRepo: userRepo} +} + +func (m *AuthMiddleware) RequireAuth(c *gin.Context) { + + //Get the bearer Token + tokenString := c.GetHeader("Authorization") + if tokenString == "" { + handleError(c, utils.ErrorUnauthorized.New("Authorization token is required")) + c.Abort() + return + } + + extractedToken := strings.Split(tokenString, "Bearer ") + + if len(extractedToken) != 2 { + handleError(c, utils.ErrorBadRequest.New("Incorrect format of authorization token")) + c.Abort() + return + } + + claims, err := m.jwtService.ParseAndValidate(strings.TrimSpace(extractedToken[1])) + if err != nil { + handleError(c, utils.ErrorBadRequest.New("Token is not valid")) + c.Abort() + return + } + + if claims.ExpiresAt != 0 && time.Now().Unix() > claims.ExpiresAt { + handleError(c, utils.ErrorUnauthorized.New("token expired")) + c.Abort() + return + } + + if claims.User, err = m.userRepo.GetByIDAndTenantID(c.Request.Context(), claims.User.ID, claims.User.TenantID); err != nil { + handleError(c, utils.ErrorUnauthorized.Wrap(err, "wrong user")) + c.Abort() + return + } + c.Request = c.Request.WithContext(context.WithValue( + c.Request.Context(), ContextParamUser, claims)) + c.Next() +} + +func GetContextIdentity(c *gin.Context) (*security.Identity, error) { + claims, ok := c.Request.Context().Value(ContextParamUser).(*security.Identity) + if !ok { + return nil, utils.ErrorPermission.New("broken session") + } + return claims, nil +} diff --git a/src/backend/core/server/router.go b/src/backend/core/server/router.go new file mode 100644 index 00000000..2466bf23 --- /dev/null +++ b/src/backend/core/server/router.go @@ -0,0 +1,87 @@ +package server + +import ( + "cognix.ch/api/v2/core/security" + "cognix.ch/api/v2/core/utils" + validation "github.com/go-ozzo/ozzo-validation/v4" + "go.uber.org/zap" + "net/http" + + "github.com/gin-gonic/gin" +) + +type HandlerFunc func(c *gin.Context) error +type HandleFuncAuth func(c *gin.Context, identity *security.Identity) error +type JsonErrorResponse struct { + Status int `json:"status,omitempty"` + Error string `json:"error,omitempty"` + OriginalError string `json:"original_error,omitempty"` +} + +type JsonResponse struct { + Status int `json:"status,omitempty"` + Error string `json:"error,omitempty"` + Data interface{} `json:"data,omitempty"` +} + +func HandlerErrorFuncAuth(f HandleFuncAuth) gin.HandlerFunc { + return func(c *gin.Context) { + identity, err := GetContextIdentity(c) + if err == nil { + err = f(c, identity) + } + handleError(c, err) + } +} + +func HandlerErrorFunc(f HandlerFunc) gin.HandlerFunc { + return func(c *gin.Context) { + handleError(c, f(c)) + } +} + +func handleError(c *gin.Context, err error) { + if err != nil { + ew, ok := err.(utils.Errors) + if !ok { + ew.Original = err + ew.Code = http.StatusInternalServerError + ew.Message = err.Error() + } + errResp := JsonErrorResponse{ + Status: int(ew.Code), + Error: ew.Message, + } + if ew.Original != nil { + errResp.OriginalError = ew.Original.Error() + } + zap.S().Errorf("[%s] %v", ew.Message, ew.Original) + c.JSON(int(ew.Code), errResp) + } +} + +func JsonResult(c *gin.Context, status int, data interface{}) error { + c.JSON(status, JsonResponse{ + Status: status, + Error: "", + Data: data, + }) + return nil +} + +func StringResult(c *gin.Context, status int, data []byte) error { + c.Data(status, "", data) + + return nil +} +func BindJsonAndValidate(c *gin.Context, data interface{}) error { + if err := c.BindJSON(data); err != nil { + return utils.ErrorBadRequest.Wrap(err, "wrong payload") + } + if vl, ok := data.(validation.Validatable); ok { + if err := vl.Validate(); err != nil { + return utils.ErrorBadRequest.New(err.Error()) + } + } + return nil +} diff --git a/src/backend/core/storage/milvus.go b/src/backend/core/storage/milvus.go new file mode 100644 index 00000000..56aab1fc --- /dev/null +++ b/src/backend/core/storage/milvus.go @@ -0,0 +1,289 @@ +package storage + +import ( + "cognix.ch/api/v2/core/utils" + "context" + "encoding/json" + "fmt" + validation "github.com/go-ozzo/ozzo-validation/v4" + milvus "github.com/milvus-io/milvus-sdk-go/v2/client" + "github.com/milvus-io/milvus-sdk-go/v2/entity" + "go.uber.org/fx" + "go.uber.org/zap" + "strconv" + "strings" + "time" +) + +const ( + ColumnNameID = "id" + ColumnNameDocumentID = "document_id" + ColumnNameContent = "content" + ColumnNameVector = "vector" + + VectorDimension = 1536 + + IndexStrategyDISKANN = "DISKANN" + IndexStrategyAUTOINDEX = "AUTOINDEX" + IndexStrategyNoIndex = "NOINDEX" +) + +var responseColumns = []string{ColumnNameID, ColumnNameDocumentID, ColumnNameContent} + +type ( + MilvusConfig struct { + Address string `env:"MILVUS_URL,required"` + Username string `env:"MILVUS_USERNAME,required"` + Password string `env:"MILVUS_PASSWORD,required"` + MetricType string `env:"MILVUS_METRIC_TYPE" envDefault:"COSINE"` + IndexStrategy string `env:"MILVUS_INDEX_STRATEGY" envDefault:"DISKANN"` + } + MilvusPayload struct { + ID int64 `json:"id"` + DocumentID int64 `json:"document_id"` + Chunk int64 `json:"chunk"` + Content string `json:"content"` + Vector []float32 `json:"vector"` + } + MilvusClient interface { + CreateSchema(ctx context.Context, name string) error + Save(ctx context.Context, collection string, payloads ...*MilvusPayload) error + Load(ctx context.Context, collection string, vector []float32) ([]*MilvusPayload, error) + Delete(ctx context.Context, collection string, documentID ...int64) error + } + milvusClient struct { + client milvus.Client + cfg *MilvusConfig + MetricType entity.MetricType + } +) + +func (c *milvusClient) Delete(ctx context.Context, collection string, documentIDs ...int64) error { + if err := c.checkConnection(); err != nil { + return err + } + var docsID []string + for _, id := range documentIDs { + docsID = append(docsID, strconv.FormatInt(id, 10)) + } + queryResult, err := c.client.Query(ctx, collection, []string{}, + fmt.Sprintf("document_id in [%s]", strings.Join(docsID, ",")), + []string{"id"}, + ) + if err != nil { + return err + } + var ids []string + for _, result := range queryResult { + for i := 0; i < result.Len(); i++ { + if id, err := result.GetAsInt64(i); err == nil { + ids = append(ids, strconv.FormatInt(id, 10)) + } + } + } + if len(ids) == 0 { + return c.client.Delete(ctx, collection, "", fmt.Sprintf("id in [%s]", strings.Join(ids, ","))) + } + return nil +} + +func (v MilvusConfig) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.IndexStrategy, validation.Required, + validation.In(IndexStrategyDISKANN, IndexStrategyAUTOINDEX, IndexStrategyNoIndex)), + validation.Field(&v.MetricType, validation.Required, + validation.In(string(entity.COSINE), string(entity.L2), string(entity.IP))), + ) +} + +func (c *milvusClient) Load(ctx context.Context, collection string, vector []float32) ([]*MilvusPayload, error) { + if err := c.checkConnection(); err != nil { + return nil, err + } + vs := []entity.Vector{entity.FloatVector(vector)} + sp, _ := entity.NewIndexFlatSearchParam() + result, err := c.client.Search(ctx, collection, []string{}, "", responseColumns, vs, ColumnNameVector, c.MetricType, 10, sp) + if err != nil { + return nil, err + } + var payload []*MilvusPayload + for _, row := range result { + for i := 0; i < row.ResultCount; i++ { + var pr MilvusPayload + if err = pr.FromResult(i, row); err != nil { + return nil, err + } + payload = append(payload, &pr) + } + } + return payload, nil +} + +var MilvusModule = fx.Options( + fx.Provide(func() (*MilvusConfig, error) { + cfg := MilvusConfig{} + if err := utils.ReadConfig(&cfg); err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return &cfg, nil + }, + NewMilvusClient, + ), +) + +func NewMilvusClient(cfg *MilvusConfig) (MilvusClient, error) { + client, err := connect(cfg) + if err != nil { + zap.S().Errorf("connect to milvus error %s ", err.Error()) + } + return &milvusClient{ + client: client, + cfg: cfg, + MetricType: entity.MetricType(cfg.MetricType), + }, nil +} + +func (c *milvusClient) Save(ctx context.Context, collection string, payloads ...*MilvusPayload) error { + var ids, documentIDs, chunks []int64 + var contents [][]byte + var vectors [][]float32 + if err := c.checkConnection(); err != nil { + return err + } + for _, payload := range payloads { + ids = append(ids, payload.ID) + documentIDs = append(documentIDs, payload.DocumentID) + chunks = append(chunks, payload.Chunk) + contents = append(contents, []byte(fmt.Sprintf(`{"content":"%s"}`, payload.Content))) + vectors = append(vectors, payload.Vector) + } + if _, err := c.client.Insert(ctx, collection, "", + entity.NewColumnInt64(ColumnNameID, ids), + entity.NewColumnInt64(ColumnNameDocumentID, documentIDs), + entity.NewColumnJSONBytes(ColumnNameContent, contents), + entity.NewColumnFloatVector(ColumnNameVector, VectorDimension, vectors), + ); err != nil { + return err + } + return nil +} + +func (c *milvusClient) indexStrategy() (entity.Index, error) { + switch c.cfg.IndexStrategy { + case IndexStrategyAUTOINDEX: + return entity.NewIndexAUTOINDEX(c.MetricType) + case IndexStrategyDISKANN: + return entity.NewIndexDISKANN(c.MetricType) + } + return nil, fmt.Errorf("index strategy %s not supported yet", c.cfg.IndexStrategy) +} + +func (c *milvusClient) CreateSchema(ctx context.Context, name string) error { + + collExists, err := c.client.HasCollection(ctx, name) + if err != nil { + return err + } + if collExists { + if err = c.client.DropCollection(ctx, name); err != nil { + return err + } + collExists = false + } + schema := entity.NewSchema().WithName(name). + WithField(entity.NewField().WithName(ColumnNameID).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)). + WithField(entity.NewField().WithName(ColumnNameDocumentID).WithDataType(entity.FieldTypeInt64)). + WithField(entity.NewField().WithName(ColumnNameContent).WithDataType(entity.FieldTypeJSON)). + WithField(entity.NewField().WithName(ColumnNameVector).WithDataType(entity.FieldTypeFloatVector).WithDim(1536)) + if err = c.client.CreateCollection(ctx, schema, 2, milvus.WithAutoID(true)); err != nil { + return err + } + + if c.cfg.IndexStrategy != IndexStrategyNoIndex { + indexStrategy, err := c.indexStrategy() + if err != nil { + return err + } + if err = c.client.CreateIndex(ctx, name, ColumnNameVector, indexStrategy, true); err != nil { + return err + } + } + return nil +} + +func (p *MilvusPayload) FromResult(i int, res milvus.SearchResult) error { + var err error + + for _, field := range res.Fields { + switch field.Name() { + case ColumnNameID: + p.ID, err = field.GetAsInt64(i) + case ColumnNameDocumentID: + p.DocumentID, err = field.GetAsInt64(i) + case ColumnNameContent: + row, err := field.GetAsString(i) + if err != nil { + continue + } + contentS := "" + if err = json.Unmarshal([]byte(row), &contentS); err == nil { + contentS = strings.ReplaceAll(contentS, "\n", "") + content := make(map[string]string) + if err = json.Unmarshal([]byte(contentS), &content); err == nil { + p.Content = content[ColumnNameContent] + } + } else { + content := make(map[string]string) + if err = json.Unmarshal([]byte(row), &content); err == nil { + p.Content = content[ColumnNameContent] + } + } + } + if err != nil { + return err + } + } + return nil +} + +// checkConnection check is milvus connection is ready +func (c *milvusClient) checkConnection() error { + // creates connection if not exists + if c.client == nil { + client, err := connect(c.cfg) + if err != nil { + zap.S().Error(err.Error()) + return fmt.Errorf("milvus is not initialized") + } + c.client = client + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // check connection status + state, err := c.client.CheckHealth(ctx) + if err != nil { + return fmt.Errorf("client.CheckHealth error %s", err.Error()) + } + if !state.IsHealthy { + return fmt.Errorf("milvus is not ready %s", strings.Join(state.Reasons, " ")) + } + + return nil +} + +func connect(cfg *MilvusConfig) (milvus.Client, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + return milvus.NewClient(ctx, milvus.Config{ + Address: cfg.Address, + Username: cfg.Username, + Password: cfg.Password, + RetryRateLimit: &milvus.RetryRateLimitOption{ + MaxRetry: 2, + MaxBackoff: 2 * time.Second, + }, + }) +} diff --git a/src/backend/core/storage/milvus_test.go b/src/backend/core/storage/milvus_test.go new file mode 100644 index 00000000..053ba1c8 --- /dev/null +++ b/src/backend/core/storage/milvus_test.go @@ -0,0 +1,22 @@ +package storage + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "strings" + "testing" +) + +func Test_ParseMilvuseContent(t *testing.T) { + src := `"{\"content\":\"\n MessagingThe core messaging capabilities of NATS includingpub-sub,request-reply, andqueue groups.Core Publish-SubscribeRequest-ReplyJSON for Message PayloadsProtobuf for Message PayloadsConcurrent Message ProcessingIterating Over Multiple SubscriptionsJetStreamAnintegrated subsystemproviding distributed persistence.Limits-based StreamInterest-based StreamWork-queue StreamPull ConsumersPull Consumer - Applying LimitsPush Consumers (legacy)Queue Push Consumers (legacy)Multi-Stream Consumption (legacy)Migration to new JetStream APIConsumer - Fetch MessagesSubject-Mapped PartitionsAuthentication and AuthorizationTopics related toauthenticationandauthorization.Programmatic NKeys and JWTsConfiguring the System AccountPrivate InboxAuth Callout - CentralizedAuth Callout - DecentralizedPrivate Inbox using JWTTopologiesExamples showcasing various deployment topologies NATS supports includingclusters,superclusters, andleaf nodes.Simple LeafnodeLeafnode with JWT AuthSuperclusterSupercluster with JetStreamSupercluster ArbiterUse CasesCross-functional examples satisfying a use case.Regional and Cross Region Streams (Supercluster)Regional and Cross Region Streams (Cluster)Key-ValueA layer on top of JetStream for utilizing a stream as\"}"` + + contentS := "" + err := json.Unmarshal([]byte(src), &contentS) + assert.Nil(t, err) + contentS = strings.ReplaceAll(contentS, "\n", "") + content := make(map[string]string) + err = json.Unmarshal([]byte(contentS), &content) + t.Log(content["content"]) + t.Log(contentS) + +} diff --git a/src/backend/core/storage/minio.go b/src/backend/core/storage/minio.go new file mode 100644 index 00000000..e206b589 --- /dev/null +++ b/src/backend/core/storage/minio.go @@ -0,0 +1,96 @@ +package storage + +import ( + "cognix.ch/api/v2/core/utils" + "context" + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "io" +) + +type ( + MinioConfig struct { + AccessKey string `env:"MINIO_ACCESS_KEY"` + SecretAccessKey string `env:"MINIO_SECRET_ACCESS_KEY"` + Endpoint string `env:"MINIO_ENDPOINT"` + UseSSL bool `env:"MINIO_USE_SSL"` + BucketName string `env:"MINIO_BUCKET_NAME"` + Region string `env:"MINIO_REGION"` + } + MinIOClient interface { + Upload(ctx context.Context, bucket, filename, contentType string, reader io.Reader) (string, string, error) + GetObject(ctx context.Context, bucket, filename string, writer io.Writer) error + DeleteObject(ctx context.Context, bucket, filename string) error + } + minIOClient struct { + Region string + client *minio.Client + } + minIOMockClient struct{} +) + +func (c *minIOClient) DeleteObject(ctx context.Context, bucket, filename string) error { + return c.client.RemoveObject(ctx, bucket, filename, minio.RemoveObjectOptions{ + ForceDelete: true, + }) +} + +func (c *minIOClient) checkOrCreateBucket(ctx context.Context, bucketName string) error { + ok, err := c.client.BucketExists(ctx, bucketName) + if err != nil { + return err + } + // create bucket if not exists + if !ok { + return c.client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{ + Region: c.Region, + }) + } + return nil +} +func (c *minIOClient) Upload(ctx context.Context, bucket, filename, contentType string, reader io.Reader) (string, string, error) { + // verify is bucket exists. create if not exists + if err := c.checkOrCreateBucket(ctx, bucket); err != nil { + return "", "", err + } + + // save file in minio + res, err := c.client.PutObject(ctx, bucket, filename, reader, -1, + minio.PutObjectOptions{ + ContentType: contentType, + NumThreads: 0, + }) + if err != nil { + return "", "", utils.Internal.Wrapf(err, "cannot upload file: %s", err.Error()) + } + return res.Key, res.ChecksumCRC32C, nil +} + +func (c *minIOClient) GetObject(ctx context.Context, bucket, filename string, writer io.Writer) error { + object, err := c.client.GetObject(ctx, bucket, filename, minio.GetObjectOptions{}) + if err != nil { + return err + } + defer object.Close() + _, err = io.Copy(writer, object) + if err != nil { + return err + } + return nil +} + +func NewMinIOClient(cfg *MinioConfig) (MinIOClient, error) { + + minioClient, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, + cfg.SecretAccessKey, ""), + Secure: cfg.UseSSL, + }) + if err != nil { + return nil, err + } + return &minIOClient{ + Region: cfg.Region, + client: minioClient, + }, nil +} diff --git a/src/backend/core/storage/module.go b/src/backend/core/storage/module.go new file mode 100644 index 00000000..696e8b38 --- /dev/null +++ b/src/backend/core/storage/module.go @@ -0,0 +1,27 @@ +package storage + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +//var MinioModule = fx.Options( +// fx.Provide(func() (*MinioConfig, error) { +// cfg := MinioConfig{} +// err := utils.ReadConfig(&cfg) +// return &cfg, err +// }, +// newMinioClient, +// ), +//) + +var MinioModule = fx.Options( + fx.Provide( + func() (*MinioConfig, error) { + cfg := MinioConfig{} + err := utils.ReadConfig(&cfg) + return &cfg, err + }, + NewMinIOClient, + ), +) diff --git a/src/backend/core/utils/config.go b/src/backend/core/utils/config.go new file mode 100644 index 00000000..e83ffb5d --- /dev/null +++ b/src/backend/core/utils/config.go @@ -0,0 +1,7 @@ +package utils + +import "github.com/caarlos0/env/v10" + +func ReadConfig(cfg interface{}) error { + return env.Parse(cfg) +} diff --git a/src/backend/core/utils/errors.go b/src/backend/core/utils/errors.go new file mode 100644 index 00000000..e899d40f --- /dev/null +++ b/src/backend/core/utils/errors.go @@ -0,0 +1,57 @@ +package utils + +import ( + "fmt" + "net/http" +) + +type Errors struct { + Code ErrorWrap + Message string + Original error +} +type ErrorWrap int + +const ( + ErrorPermission ErrorWrap = http.StatusForbidden + NotFound ErrorWrap = http.StatusNotFound + Internal ErrorWrap = http.StatusInternalServerError + ErrorBadRequest ErrorWrap = http.StatusBadRequest + ErrorUnauthorized ErrorWrap = http.StatusUnauthorized +) + +func (e Errors) Error() string { + return e.Message +} + +func (e ErrorWrap) Wrap(eo error, msg string) Errors { + return Errors{ + Code: e, + Message: msg, + Original: eo, + } +} + +func (e ErrorWrap) Wrapf(eo error, msg string, args ...interface{}) Errors { + return Errors{ + Code: e, + Message: fmt.Sprintf(msg, args...), + Original: eo, + } +} + +func (e ErrorWrap) New(msg string) Errors { + return Errors{ + Code: e, + Message: msg, + Original: nil, + } +} + +func (e ErrorWrap) Newf(msg string, args ...interface{}) Errors { + return Errors{ + Code: e, + Message: fmt.Sprintf(msg, args...), + Original: nil, + } +} diff --git a/src/backend/core/utils/grpc.go b/src/backend/core/utils/grpc.go new file mode 100644 index 00000000..afd9b543 --- /dev/null +++ b/src/backend/core/utils/grpc.go @@ -0,0 +1,96 @@ +package utils + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + + "google.golang.org/grpc/credentials" +) + +type ( + // TLSConfig is a type that represents + // common TLS configuration + TLSConfig struct { + CACertificate string + CertificateFile string + KeyFile string + } + // credentialsConstructor is a type + // that represents credentials creator function signature + credentialsConstructor func( + certPool *x509.CertPool, certificate *tls.Certificate, + ) (credentials.TransportCredentials, error) + // certificatePoolConstructor is a type + // that represents certificate pool creator function signature + certificatePoolConstructor func( + caCertFile, certFile, keyFile []byte, + ) (*x509.CertPool, *tls.Certificate, error) +) + +// NewServerTLSCredentials creates a new instance of the TransportCredentials. +func NewServerTLSCredentials(cfg *TLSConfig) (credentials.TransportCredentials, error) { + return withCertificatePool(cfg, + newTLSCertificateWithPool, + func(certPool *x509.CertPool, certificate *tls.Certificate) (credentials.TransportCredentials, error) { + return credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{*certificate}, + ClientCAs: certPool, + }), nil + }) +} + +// NewClientTLSCredentials creates a new instance of the TransportCredentials. +func NewClientTLSCredentials(cfg *TLSConfig) (credentials.TransportCredentials, error) { + return withCertificatePool(cfg, + newTLSCertificateWithPool, + func(certPool *x509.CertPool, certificate *tls.Certificate) (credentials.TransportCredentials, error) { + return credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{*certificate}, + RootCAs: certPool, + }), nil + }) +} + +// withCertificatePool decorates constructor with certificates preloading +func withCertificatePool( + cfg *TLSConfig, + constructCertificate certificatePoolConstructor, + constructCredentials credentialsConstructor, +) (credentials.TransportCredentials, error) { + caRootCert, err := ioutil.ReadFile(cfg.CACertificate) + if err != nil { + return nil, err + } + certPEMBlock, err := ioutil.ReadFile(cfg.CACertificate) + if err != nil { + return nil, err + } + keyPEMBlock, err := ioutil.ReadFile(cfg.CACertificate) + if err != nil { + return nil, err + } + certPool, certificate, err := constructCertificate(caRootCert, certPEMBlock, keyPEMBlock) + if err != nil { + return nil, err + } + return constructCredentials(certPool, certificate) +} + +// newTLSCertificateWithPool creates a new pair of CertPool and Certificate. +func newTLSCertificateWithPool(caCertFile, certFile, keyFile []byte) (*x509.CertPool, *tls.Certificate, error) { + certificate, err := tls.X509KeyPair(certFile, keyFile) + if err != nil { + return nil, nil, err + } + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(caCertFile) + if !ok { + return nil, nil, fmt.Errorf("failed to append certs") + } + return certPool, &certificate, nil +} diff --git a/src/backend/core/utils/log.go b/src/backend/core/utils/log.go new file mode 100644 index 00000000..88edc953 --- /dev/null +++ b/src/backend/core/utils/log.go @@ -0,0 +1,30 @@ +package utils + +import ( + "github.com/uptrace/opentelemetry-go-extra/otelzap" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "os" +) + +func InitLogger(debug bool) { + + pe := zap.NewProductionEncoderConfig() + + pe.EncodeTime = zapcore.ISO8601TimeEncoder + consoleEncoder := zapcore.NewConsoleEncoder(pe) + + level := zap.InfoLevel + if debug { + level = zap.DebugLevel + } + + core := zapcore.NewTee( + zapcore.NewCore(consoleEncoder, zapcore.AddSync(os.Stdout), level), + ) + + l := zap.New(core) + zap.ReplaceGlobals(l) + otelzap.ReplaceGlobals(otelzap.New(l)) + +} diff --git a/src/backend/core/utils/map.go b/src/backend/core/utils/map.go new file mode 100644 index 00000000..09458cd3 --- /dev/null +++ b/src/backend/core/utils/map.go @@ -0,0 +1,30 @@ +package utils + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Map map[string]interface{} + +func (m Map) Value() (driver.Value, error) { + j, err := json.Marshal(m) + return string(j), err +} + +func (m *Map) Scan(val interface{}) error { + value, ok := val.([]byte) + if !ok { + return fmt.Errorf("Type assertion .([]byte) failed.") + } + return json.Unmarshal(value, m) +} + +func (m *Map) ToStruct(s interface{}) error { + buf, err := json.Marshal(m) + if err != nil { + return err + } + return json.Unmarshal(buf, s) +} diff --git a/src/backend/core/utils/resty.go b/src/backend/core/utils/resty.go new file mode 100644 index 00000000..25b538b4 --- /dev/null +++ b/src/backend/core/utils/resty.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "github.com/go-resty/resty/v2" +) + +func WrapRestyError(resp *resty.Response, err error) error { + if err != nil { + return err + } + if !resp.IsError() { + return nil + } + errMsg := string(resp.Body()) + if errMsg != "" { + return fmt.Errorf(errMsg) + } + return nil +} diff --git a/src/backend/core/utils/string.go b/src/backend/core/utils/string.go new file mode 100644 index 00000000..b9fafbd0 --- /dev/null +++ b/src/backend/core/utils/string.go @@ -0,0 +1,7 @@ +package utils + +import "strings" + +func StripFileName(filename string) string { + return strings.ReplaceAll(strings.ReplaceAll(filename, ":", ""), "-", "_") +} diff --git a/src/backend/embedder/.DS_Store b/src/backend/embedder/.DS_Store new file mode 100644 index 00000000..eb6555f4 Binary files /dev/null and b/src/backend/embedder/.DS_Store differ diff --git a/src/backend/embedder/.dockerignore b/src/backend/embedder/.dockerignore new file mode 100644 index 00000000..4c195866 --- /dev/null +++ b/src/backend/embedder/.dockerignore @@ -0,0 +1,2 @@ +models/ +__pycache__/ \ No newline at end of file diff --git a/src/backend/embedder/.env.example b/src/backend/embedder/.env.example new file mode 100644 index 00000000..be9754fe --- /dev/null +++ b/src/backend/embedder/.env.example @@ -0,0 +1,6 @@ +LOG_LEVEL=INFO +LOG_FORMAT=%(asctime)s - %(name)s - %(levelname)s - %(message)s +# max number of sentence transformer instances, could be 1gb+ each +MODEL_CACHE_LIMIT=2 +LOCAL_MODEL_PATH=../../../data/models +GRPC_PORT=50051 \ No newline at end of file diff --git a/src/backend/embedder/Dockerfile_cpu b/src/backend/embedder/Dockerfile_cpu new file mode 100644 index 00000000..9b50dab0 --- /dev/null +++ b/src/backend/embedder/Dockerfile_cpu @@ -0,0 +1,199 @@ +# ################################### +# Distroless working +# ################################### +FROM python:3.11.7-slim AS build-env + +WORKDIR /app +ADD ./ ./ + +RUN pip3 install --upgrade pip && \ + pip install -r ./requirements.txt + +# Install any needed packages specified in requirements.txt +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt \ + && rm -rf /root/.cache + +# Install dependencies and clean up +RUN apt-get update && apt-get install -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + +# the container before this 3 lines is 107mb, after it's 982mb!!!! +RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +RUN pip install transformers==4.40.2 tqdm==4.66.4 numpy==1.26.4 scikit-learn==1.4.2 scipy==1.13.0 nltk==3.8.1 pillow==10.3.0 +RUN pip install --no-deps sentence-transformers==2.7.0 + +FROM gcr.io/distroless/python3-debian12 +COPY --from=build-env /app /app +COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages + + +WORKDIR /app +ENV PYTHONPATH=/usr/local/lib/python3.11/site-packages +CMD ["embedder_service.py"] + + + + + + + +# # ################################### +# # python slim working +# # ################################### +# # using Python 3.11.0 as the newr supported by PyTorch +# FROM python:3.11.7-slim-bookworm + + +# # Set the working directory inside the container +# WORKDIR /app + +# # Copy the requirements.txt first to leverage Docker cache +# COPY requirements.txt . + +# # Install dependencies and clean up +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# build-essential \ +# && rm -rf /var/lib/apt/lists/* +# RUN pip install --upgrade pip +# RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# RUN pip install --no-cache-dir -r requirements.txt \ +# && rm -rf /root/.cache + +# # Copy the rest of the embedder application code +# COPY . . + +# # Command to run your application +# CMD ["python", "embedder_service.py"] + + + + + + + + + + + + + + +# # Use a multi-stage build to ensure the final image has the correct architecture +# # Stage 1: Build stage +# FROM --platform=linux/amd64 python:3.11-slim as build-env + +# # Set the working directory inside the container +# WORKDIR /app + +# USER root +# # Install necessary system packages for grpcio +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# build-essential \ +# libssl-dev \ +# libffi-dev \ +# python3-dev \ +# python3.11-distutils \ +# zlib1g \ +# zlib1g-dev \ +# libexpat1-dev \ +# && rm -rf /var/lib/apt/lists/* + +# # Copy the requirements.txt first to leverage Docker cache +# COPY requirements.txt . + +# # Install dependencies and clean up +# RUN pip install --upgrade pip +# RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# RUN pip install --no-cache-dir --use-deprecated=legacy-resolver -r requirements.txt \ +# && rm -rf /root/.cache + +# # Copy the rest of the embedder application code +# COPY . /app + +# # Stage 2: Production stage +# FROM --platform=linux/amd64 gcr.io/distroless/python3-debian12 + +# # Set the working directory inside the container +# WORKDIR /app + +# # Copy the application from the build stage +# COPY --from=build-env /app /app + + +# # Required by lots of packages - e.g. six, numpy, wsgi +# COPY --from=build-env /lib/x86_64-linux-gnu/libz.so.1 /lib/x86_64-linux-gnu/ +# # Required by google-cloud/grpcio +# COPY --from=build-env /usr/lib/x86_64-linux-gnu/libffi* /usr/lib/x86_64-linux-gnu/ +# COPY --from=build-env /lib/x86_64-linux-gnu/libexpat* /lib/x86_64-linux-gnu/ +# COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.9/site-packages/ +# COPY --from=build-env /usr/local/lib/python3.11/site-packages/grpc /usr/local/lib/python3.9/site-packages/grpc/ + + +# # Set the environment variable for Python path +# ENV PYTHONPATH=/app + +# # Command to run your application +# CMD ["embedder.py"] + + + + + + + + + + + + +# # # Use a slim Python base image +# # # using Python 3.11.0 as the newr supported by PyTorch +# # FROM python:3.11.7-slim-bookworm as build-env + + +# # # Set the working directory inside the container +# # WORKDIR /app + +# # # Copy the requirements.txt first to leverage Docker cache +# # COPY requirements.txt . + +# # # Install dependencies and clean up +# # RUN apt-get update && apt-get install -y --no-install-recommends \ +# # build-essential \ +# # && rm -rf /var/lib/apt/lists/* +# # RUN pip install --upgrade pip +# # RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# # RUN pip install --no-cache-dir -r requirements.txt \ +# # && rm -rf /root/.cache + +# # # Copy the rest of the embedder application code +# # COPY . . + +# # # Command to run your application + +# # # Stage 2: Production stage +# # FROM --platform=linux/amd64 gcr.io/distroless/python3-debian12 + +# # COPY --from=build-env /app /app +# # # Required by lots of packages - e.g. six, numpy, wsgi +# # COPY --from=build-env /lib/x86_64-linux-gnu/libz.so.1 /lib/x86_64-linux-gnu/ +# # # Required by google-cloud/grpcio +# # COPY --from=build-env /usr/lib/x86_64-linux-gnu/libffi* /usr/lib/x86_64-linux-gnu/ +# # COPY --from=build-env /lib/x86_64-linux-gnu/libexpat* /lib/x86_64-linux-gnu/ + +# # #COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +# # WORKDIR /app +# # ENV PYTHONPATH=/app +# # CMD ["embedder.py"] + + + + + + + + +# # crazy +# # https://alex-moss.medium.com/creating-an-up-to-date-python-distroless-container-image-e3da728d7a80 +# # \ No newline at end of file diff --git a/src/backend/embedder/Dockerfile_gpu b/src/backend/embedder/Dockerfile_gpu new file mode 100644 index 00000000..daaa0a6e --- /dev/null +++ b/src/backend/embedder/Dockerfile_gpu @@ -0,0 +1,200 @@ +# ################################### +# Distroless working +# ################################### +FROM python:3.11.7-slim AS build-env + +WORKDIR /app +ADD ./ ./ + +RUN pip3 install --upgrade pip && \ + pip install -r ./requirements.txt + +# Install any needed packages specified in requirements.txt +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt \ + && rm -rf /root/.cache + +# Install dependencies and clean up +RUN apt-get update && apt-get install -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + +# the container before this 3 lines is 107mb, after it's 982mb!!!! +#RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +#RUN pip install transformers==4.40.2 tqdm==4.66.4 numpy==1.26.4 scikit-learn==1.4.2 scipy==1.13.0 nltk==3.8.1 pillow==10.3.0 +#RUN pip install --no-deps sentence-transformers==2.7.0 +RUN pip install sentence-transformers==2.7.0 + +FROM gcr.io/distroless/python3-debian12 +COPY --from=build-env /app /app +COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages + + +WORKDIR /app +ENV PYTHONPATH=/usr/local/lib/python3.11/site-packages +CMD ["embedder_service.py"] + + + + + + + +# # ################################### +# # python slim working +# # ################################### +# # using Python 3.11.0 as the newr supported by PyTorch +# FROM python:3.11.7-slim-bookworm + + +# # Set the working directory inside the container +# WORKDIR /app + +# # Copy the requirements.txt first to leverage Docker cache +# COPY requirements.txt . + +# # Install dependencies and clean up +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# build-essential \ +# && rm -rf /var/lib/apt/lists/* +# RUN pip install --upgrade pip +# RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# RUN pip install --no-cache-dir -r requirements.txt \ +# && rm -rf /root/.cache + +# # Copy the rest of the embedder application code +# COPY . . + +# # Command to run your application +# CMD ["python", "embedder_service.py"] + + + + + + + + + + + + + + +# # Use a multi-stage build to ensure the final image has the correct architecture +# # Stage 1: Build stage +# FROM --platform=linux/amd64 python:3.11-slim as build-env + +# # Set the working directory inside the container +# WORKDIR /app + +# USER root +# # Install necessary system packages for grpcio +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# build-essential \ +# libssl-dev \ +# libffi-dev \ +# python3-dev \ +# python3.11-distutils \ +# zlib1g \ +# zlib1g-dev \ +# libexpat1-dev \ +# && rm -rf /var/lib/apt/lists/* + +# # Copy the requirements.txt first to leverage Docker cache +# COPY requirements.txt . + +# # Install dependencies and clean up +# RUN pip install --upgrade pip +# RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# RUN pip install --no-cache-dir --use-deprecated=legacy-resolver -r requirements.txt \ +# && rm -rf /root/.cache + +# # Copy the rest of the embedder application code +# COPY . /app + +# # Stage 2: Production stage +# FROM --platform=linux/amd64 gcr.io/distroless/python3-debian12 + +# # Set the working directory inside the container +# WORKDIR /app + +# # Copy the application from the build stage +# COPY --from=build-env /app /app + + +# # Required by lots of packages - e.g. six, numpy, wsgi +# COPY --from=build-env /lib/x86_64-linux-gnu/libz.so.1 /lib/x86_64-linux-gnu/ +# # Required by google-cloud/grpcio +# COPY --from=build-env /usr/lib/x86_64-linux-gnu/libffi* /usr/lib/x86_64-linux-gnu/ +# COPY --from=build-env /lib/x86_64-linux-gnu/libexpat* /lib/x86_64-linux-gnu/ +# COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.9/site-packages/ +# COPY --from=build-env /usr/local/lib/python3.11/site-packages/grpc /usr/local/lib/python3.9/site-packages/grpc/ + + +# # Set the environment variable for Python path +# ENV PYTHONPATH=/app + +# # Command to run your application +# CMD ["embedder.py"] + + + + + + + + + + + + +# # # Use a slim Python base image +# # # using Python 3.11.0 as the newr supported by PyTorch +# # FROM python:3.11.7-slim-bookworm as build-env + + +# # # Set the working directory inside the container +# # WORKDIR /app + +# # # Copy the requirements.txt first to leverage Docker cache +# # COPY requirements.txt . + +# # # Install dependencies and clean up +# # RUN apt-get update && apt-get install -y --no-install-recommends \ +# # build-essential \ +# # && rm -rf /var/lib/apt/lists/* +# # RUN pip install --upgrade pip +# # RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# # RUN pip install --no-cache-dir -r requirements.txt \ +# # && rm -rf /root/.cache + +# # # Copy the rest of the embedder application code +# # COPY . . + +# # # Command to run your application + +# # # Stage 2: Production stage +# # FROM --platform=linux/amd64 gcr.io/distroless/python3-debian12 + +# # COPY --from=build-env /app /app +# # # Required by lots of packages - e.g. six, numpy, wsgi +# # COPY --from=build-env /lib/x86_64-linux-gnu/libz.so.1 /lib/x86_64-linux-gnu/ +# # # Required by google-cloud/grpcio +# # COPY --from=build-env /usr/lib/x86_64-linux-gnu/libffi* /usr/lib/x86_64-linux-gnu/ +# # COPY --from=build-env /lib/x86_64-linux-gnu/libexpat* /lib/x86_64-linux-gnu/ + +# # #COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +# # WORKDIR /app +# # ENV PYTHONPATH=/app +# # CMD ["embedder.py"] + + + + + + + + +# # crazy +# # https://alex-moss.medium.com/creating-an-up-to-date-python-distroless-container-image-e3da728d7a80 +# # \ No newline at end of file diff --git a/src/backend/embedder/__init__.py b/src/backend/embedder/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/embedder/commands.txt b/src/backend/embedder/commands.txt new file mode 100644 index 00000000..34e7e23b --- /dev/null +++ b/src/backend/embedder/commands.txt @@ -0,0 +1,64 @@ + + +# build a container +sudo docker build -t cognix/embedder:latest + +# run container +docker run --name embedder cognix/embedder:latest + +#log inside a container +docker exec -it CONTAINER_NAME sh + +df + +# interactive shell +docker run -it --entrypoint /bin/sh cognix/embedder:latest + +# cleans a lot of space +docker system prune + + +from cognix-services +sudo docker-compose -f deployment/docker-compose-services-ai.yaml down +sudo docker-compose -f deployment/docker-compose-services-ai.yaml up + + +es, it is possible to create symbolic links (symlinks) in your file system, and Visual Studio Code (VS Code) will treat them as regular files. This allows you to reference files located in different directories without moving the actual files. + +Here's how you can create a symlink for a file or directory: + + +Visual Studio Code terminal shows multiple Conda envs +execute conda config --set auto_activate_base False or +turn off "Activate Python Environment in all Terminals created" setting in the Python extension configuration in VSCode. + +To generate python objects out of proto files +Attention this is the right command I got sever troubles +python -m grpc_tools.protoc -I . --python_out=. --pyi_out=. --grpc_python_out=. protos/embed_service.proto + +python3 -m grpc_tools.protoc -I . --python_out=. --pyi_out=. --grpc_python_out=. embedd_service.proto + + + + + + +pod configuration + +apiVersion: v1 +kind: Pod +metadata: + name: sentence-transformer +spec: + containers: + - name: sentence-transformer + image: your-docker-image + resources: + requests: + memory: "4Gi" # Start with 4GiB for a medium-sized model + cpu: "2000m" # 2 vCPUs + limits: + memory: "8Gi" # Allow up to 8GiB for peak usage + cpu: "4000m" # Allow up to 4 vCPUs + + diff --git a/src/backend/embedder/embedder-deployment.yaml b/src/backend/embedder/embedder-deployment.yaml new file mode 100644 index 00000000..a9fc2a3b --- /dev/null +++ b/src/backend/embedder/embedder-deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: embedder +spec: + replicas: 1 + selector: + matchLabels: + app: embedder + template: + metadata: + labels: + app: embedder + spec: + containers: + - name: embedder + image: cognixacr.azurecr.io/cognix/embedder:main + envFrom: + - configMapRef: + name: embedder-srv + ports: + - containerPort: 50051 + volumeMounts: + - mountPath: /models + name: models + imagePullPolicy: Always + imagePullSecrets: + - name: pullsecret + volumes: + - name: models + persistentVolumeClaim: + claimName: models +--- +apiVersion: v1 +kind: Service +metadata: + name: embedder +spec: + selector: + app: embedder + ports: + - protocol: TCP + port: 50051 + targetPort: 50051 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: models + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: azurefile-csi diff --git a/src/backend/embedder/embedder_service.py b/src/backend/embedder/embedder_service.py new file mode 100644 index 00000000..854ffdb1 --- /dev/null +++ b/src/backend/embedder/embedder_service.py @@ -0,0 +1,75 @@ +import time +import os + +from lib.gen_types.embed_service_pb2_grpc import EmbedServiceServicer, add_EmbedServiceServicer_to_server +from lib.gen_types.embed_service_pb2 import EmbedResponse +from sentence_encoder import SentenceEncoder +from lib.helpers.device_checker import DeviceChecker +import grpc +from concurrent import futures +import logging +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +# Get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) + +# Get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + +# Get gRPC port from environment variable +grpc_port = os.getenv('GRPC_PORT', '50051') + + +class EmbedServicer(EmbedServiceServicer): + def GetEmbeding(self, request, context): + start_time = time.time() # Record the start time + try: + logger.info(f"incoming embedd request: {request}") + embed_response = EmbedResponse() + + encoded_data = SentenceEncoder.embed(text=request.content, model_name=request.model) + + # assign the vector variable the response + embed_response.vector.extend(encoded_data) + logger.info("embedd request successfully processed") + return embed_response + except Exception as e: + logger.exception(e) + raise grpc.RpcError(f"❌ failed to process request: {str(e)}") + finally: + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + logger.info(f"⏰ total elapsed time: {elapsed_time:.2f} seconds") + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(), + options=[ + ('grpc.max_send_message_length', 100 * 1024 * 1024), # 100 MB + ('grpc.max_receive_message_length', 100 * 1024 * 1024) # 100 MB + ] + ) + + # Pass the readiness_probe to EmbedServicer + # embed_servicer = EmbedServicer(readiness_probe) + # add_EmbedServiceServicer_to_server(embed_servicer, server) + + add_EmbedServiceServicer_to_server(EmbedServicer(), server) + + server.add_insecure_port(f"0.0.0.0:{grpc_port}") + server.start() + logger.info(f"👂 embedder listening on port {grpc_port}") + DeviceChecker.check_device() + server.wait_for_termination() + + +if __name__ == "__main__": + serve() diff --git a/src/backend/embedder/lib/gen_types/__init__.py b/src/backend/embedder/lib/gen_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/embedder/lib/gen_types/embed_service_pb2.py b/src/backend/embedder/lib/gen_types/embed_service_pb2.py new file mode 100644 index 00000000..f964e15a --- /dev/null +++ b/src/backend/embedder/lib/gen_types/embed_service_pb2.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: embed_service.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x65mbed_service.proto\x12\ncom.cognix\".\n\x0c\x45mbedRequest\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\"\x1f\n\rEmbedResponse\x12\x0e\n\x06vector\x18\x01 \x03(\x02\x32T\n\x0c\x45mbedService\x12\x44\n\x0bGetEmbeding\x12\x18.com.cognix.EmbedRequest\x1a\x19.com.cognix.EmbedResponse\"\x00\x42\x1aZ\x18\x62\x61\x63kend/core/proto;protob\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embed_service_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z\030backend/core/proto;proto' + _globals['_EMBEDREQUEST']._serialized_start=35 + _globals['_EMBEDREQUEST']._serialized_end=81 + _globals['_EMBEDRESPONSE']._serialized_start=83 + _globals['_EMBEDRESPONSE']._serialized_end=114 + _globals['_EMBEDSERVICE']._serialized_start=116 + _globals['_EMBEDSERVICE']._serialized_end=200 +# @@protoc_insertion_point(module_scope) diff --git a/src/backend/embedder/lib/gen_types/embed_service_pb2.pyi b/src/backend/embedder/lib/gen_types/embed_service_pb2.pyi new file mode 100644 index 00000000..d8baa1dc --- /dev/null +++ b/src/backend/embedder/lib/gen_types/embed_service_pb2.pyi @@ -0,0 +1,20 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class EmbedRequest(_message.Message): + __slots__ = ("content", "model") + CONTENT_FIELD_NUMBER: _ClassVar[int] + MODEL_FIELD_NUMBER: _ClassVar[int] + content: str + model: str + def __init__(self, content: _Optional[str] = ..., model: _Optional[str] = ...) -> None: ... + +class EmbedResponse(_message.Message): + __slots__ = ("vector",) + VECTOR_FIELD_NUMBER: _ClassVar[int] + vector: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, vector: _Optional[_Iterable[float]] = ...) -> None: ... diff --git a/src/backend/embedder/lib/gen_types/embed_service_pb2_grpc.py b/src/backend/embedder/lib/gen_types/embed_service_pb2_grpc.py new file mode 100644 index 00000000..c23c9380 --- /dev/null +++ b/src/backend/embedder/lib/gen_types/embed_service_pb2_grpc.py @@ -0,0 +1,106 @@ + +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! + + + + +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import lib.gen_types.embed_service_pb2 as embed__service__pb2 + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in embed_service_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) + + +class EmbedServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetEmbeding = channel.unary_unary( + '/com.cognix.EmbedService/GetEmbeding', + request_serializer=embed__service__pb2.EmbedRequest.SerializeToString, + response_deserializer=embed__service__pb2.EmbedResponse.FromString, + _registered_method=True) + + +class EmbedServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def GetEmbeding(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_EmbedServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetEmbeding': grpc.unary_unary_rpc_method_handler( + servicer.GetEmbeding, + request_deserializer=embed__service__pb2.EmbedRequest.FromString, + response_serializer=embed__service__pb2.EmbedResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.cognix.EmbedService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class EmbedService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def GetEmbeding(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/com.cognix.EmbedService/GetEmbeding', + embed__service__pb2.EmbedRequest.SerializeToString, + embed__service__pb2.EmbedResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/backend/embedder/lib/helpers/device_checker.py b/src/backend/embedder/lib/helpers/device_checker.py new file mode 100644 index 00000000..97872514 --- /dev/null +++ b/src/backend/embedder/lib/helpers/device_checker.py @@ -0,0 +1,62 @@ +import os + +import torch +import logging + +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +# Get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) + +# Get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + +class DeviceChecker: + logger = logger + + @staticmethod + def check_device(): + DeviceChecker.logger.info(f"🤖🤖🤖🤖🤖🤖 PyTorch version: {torch.__version__}") + + # Check PyTorch has access to CUDA (NVIDIA's GPU architecture) + cuda_available = torch.cuda.is_available() + + # Check PyTorch has access to MPS (Metal Performance Shader, Apple's GPU architecture) + mps_built = torch.backends.mps.is_built() + mps_available = torch.backends.mps.is_available() + + # Determine the device to use + if cuda_available: + device = "cuda" + elif mps_available: + device = "mps" + else: + device = "cpu" + + DeviceChecker.logger.info(f"🤖🤖🤖🤖🤖🤖 Using device: {device}") + + # Display additional information based on the device + # if device == "cuda": + # cuda_version = torch.version.cuda + # DeviceChecker.logger.info(f"Is CUDA available? {cuda_available}") + # DeviceChecker.logger.info(f"CUDA version: {cuda_version}") + # elif device == "mps": + # DeviceChecker.logger.info(f"Is MPS (Metal Performance Shader) built? {mps_built}") + # DeviceChecker.logger.info(f"Is MPS available? {mps_available}") + + # Create data and send it to the device + x = torch.rand(size=(3, 4)).to(device) + # DeviceChecker.logger.info(f"Tensor on {device}: {x}") + + +# # To test the device checker +# if __name__ == "__main__": +# DeviceChecker.check_device() diff --git a/src/backend/embedder/requirements.txt b/src/backend/embedder/requirements.txt new file mode 100644 index 00000000..b38eee77 --- /dev/null +++ b/src/backend/embedder/requirements.txt @@ -0,0 +1,18 @@ +nats-py==2.7.2 +#protobuf==4.25.3 +python-dotenv==1.0.1 +protobuf==5.27.0 +grpcio==1.63.0 +grpcio-tools==1.63.0 +#sentence-transformers==2.7.0 + + +# --find-links https://download.pytorch.org/whl/torch_stable.html +# torch==2.3.0 +## move to torch+cpu to minize a lot +# torch==2.3.0+cpu installed directly in dockerfile +# #torchvision==2.3.0+cpu +# #torchaudio==2.3.0+cpu +#-f https://download.pytorch.org/whl/torch_stable.html + + diff --git a/src/backend/embedder/sentence_encoder.py b/src/backend/embedder/sentence_encoder.py new file mode 100644 index 00000000..1542a97c --- /dev/null +++ b/src/backend/embedder/sentence_encoder.py @@ -0,0 +1,107 @@ +import os +from sentence_transformers import SentenceTransformer +import logging +import threading +from typing import Dict, List +from dotenv import load_dotenv + +# compare performance with https://github.com/qdrant/fastembed + +# Load environment variables from .env file +load_dotenv() + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +class SentenceEncoder: + # Validate environment variables at class level initialization + _cache_limit: int = int(os.getenv('MODEL_CACHE_LIMIT', 1)) + _local_model_dir: str = os.getenv('LOCAL_MODEL_PATH', 'models') + + if _cache_limit <= 0: + raise ValueError("MODEL_CACHE_LIMIT must be an integer greater than 0") + + # Convert the local model path to an absolute path based on the working directory + _local_model_dir = os.path.abspath(_local_model_dir) + + if not os.path.isdir(_local_model_dir): + raise ValueError(f"LOCAL_MODEL_PATH '{_local_model_dir}' is not a valid directory") + + # Thread lock for thread-safe access to the cache + _lock: threading.Lock = threading.Lock() + + # Dictionary to store cached model instances + _model_cache: Dict[str, SentenceTransformer] = {} + + + @classmethod + def _load_model(cls, model_name: str) -> SentenceTransformer: + """ + Loads a model from the local directory if available, otherwise downloads and saves it. + + Parameters: + model_name (str): The name of the model to load or download. + + Returns: + SentenceTransformer: The loaded SentenceTransformer model. + """ + model_path: str = os.path.join(cls._local_model_dir, model_name) + + if not os.path.exists(model_path) or not os.listdir(model_path): + logger.info(f"{model_name} model not found locally, downloading from Hugging Face...") + try: + model: SentenceTransformer = SentenceTransformer(model_name) + model.save(model_path) + logger.info(f"{model_name} model saved locally at {model_path}") + except Exception as e: + logger.info(f"❌ {model_name} failed to download or save the model due to: {e}") + else: + logger.info(f"loading {model_name} from local directory...") + + return SentenceTransformer(model_path) + + @classmethod + def _get_model(cls, model_name: str) -> SentenceTransformer: + """ + Retrieves a model from the cache or loads it if not already cached. Manages the cache size. + + Parameters: + model_name (str): The name of the model to retrieve. + + Returns: + SentenceTransformer: The model instance. + """ + with cls._lock: + # Check if the model is already in the cache + if model_name in cls._model_cache: + logger.info(f"using cached model: {model_name}") + return cls._model_cache[model_name] + + # If the cache limit is reached, unload the oldest model + if len(cls._model_cache) >= cls._cache_limit: + oldest_model: str = next(iter(cls._model_cache)) + logger.info(f"unloading model: {oldest_model}") + # removing model from cache and memory + del cls._model_cache[oldest_model] + + # Load and cache the new model + logger.info(f"loading model: {model_name}") + model: SentenceTransformer = cls._load_model(model_name) + cls._model_cache[model_name] = model + return model + + @classmethod + def embed(cls, text: str, model_name: str) -> List[float]: + """ + Encodes the provided text using the specified model. + + Parameters: + text (str): The text to be encoded. + model_name (str): The name of the model to use for encoding. + + Returns: + list: A list of floats representing the encoded text. + """ + model: SentenceTransformer = cls._get_model(model_name) + return model.encode(text).tolist() \ No newline at end of file diff --git a/src/backend/embedder/test.py b/src/backend/embedder/test.py new file mode 100644 index 00000000..a3018d61 --- /dev/null +++ b/src/backend/embedder/test.py @@ -0,0 +1,17 @@ +import asyncio +import logging + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +async def main(): + try: + while True: + await asyncio.sleep(1) + logger.info("embedder is working") + except KeyboardInterrupt: + logger.info("keyb interrupt") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/src/backend/embedder/test_embedder.py b/src/backend/embedder/test_embedder.py new file mode 100644 index 00000000..0b24bfeb --- /dev/null +++ b/src/backend/embedder/test_embedder.py @@ -0,0 +1,36 @@ +from lib.gen_types.embed_service_pb2_grpc import EmbedServiceStub +from lib.gen_types.embed_service_pb2 import EmbedRequest +import grpc + +def run(): + #with grpc.insecure_channel('127.0.0.1:50051') as channel: + with grpc.insecure_channel('localhost:50051') as channel: + stub = EmbedServiceStub(channel) + print("Calling gRPC Service GetEmbed - Unary") + + content_to_embedd = input("type the content you want to embedd: ") + + embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2") + embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="microsoft/mpnet-base") + # embed_response = stub.GetEmbeding(embed_request) + # + # embed_request = EmbedRequest(content=content_to_embedd, model="distilbert/distilroberta-base") + # embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2") + # embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/natural-questions") + # embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/wikianswers-duplicates") + # embed_response = stub.GetEmbeding(embed_request) + + + print("GetEmbed Response Received:") + print(embed_response.vector) + +if __name__ == "__main__": + run() diff --git a/src/backend/embedder/test_embedder_recursive.py b/src/backend/embedder/test_embedder_recursive.py new file mode 100644 index 00000000..401c0b9d --- /dev/null +++ b/src/backend/embedder/test_embedder_recursive.py @@ -0,0 +1,36 @@ +from lib.gen_types.embed_service_pb2_grpc import EmbedServiceStub +from lib.gen_types.embed_service_pb2 import EmbedRequest +import grpc + +def run(): + #with grpc.insecure_channel('127.0.0.1:50051') as channel: + with grpc.insecure_channel('localhost:50051') as channel: + stub = EmbedServiceStub(channel) + print("Calling gRPC Service GetEmbed - Unary") + + for i in range(10000): + content_to_embedd = "docker-compose up --build – Jinna Baalu Apr 22, 2019 at 14:5 Just to clarify upper comment: docker-compose up --build rebuild all containers. Use docker-compose up --build as stated in @denov comment. – J" + embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2") + embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="microsoft/mpnet-base") + # embed_response = stub.GetEmbeding(embed_request) + # + # embed_request = EmbedRequest(content=content_to_embedd, model="distilbert/distilroberta-base") + # embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2") + # embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/natural-questions") + # embed_response = stub.GetEmbeding(embed_request) + + # embed_request = EmbedRequest(content=content_to_embedd, model="sentence-transformers/wikianswers-duplicates") + # embed_response = stub.GetEmbeding(embed_request) + + + print("GetEmbed Response Received:") + # print(embed_response.vector) + +if __name__ == "__main__": + run() diff --git a/src/backend/embedder/test_gpu.py b/src/backend/embedder/test_gpu.py new file mode 100644 index 00000000..76ebde1b --- /dev/null +++ b/src/backend/embedder/test_gpu.py @@ -0,0 +1,33 @@ +import torch + +print(f"PyTorch version: {torch.__version__}") + +# Check PyTorch has access to CUDA (NVIDIA's GPU architecture) +cuda_available = torch.cuda.is_available() + +# Check PyTorch has access to MPS (Metal Performance Shader, Apple's GPU architecture) +mps_built = torch.backends.mps.is_built() +mps_available = torch.backends.mps.is_available() + +# Determine the device to use +if cuda_available: + device = "cuda" +elif mps_available: + device = "mps" +else: + device = "cpu" + +print(f"Using device: {device}") + +# Display additional information based on the device +if device == "cuda": + cuda_version = torch.version.cuda + print(f"Is CUDA available? {cuda_available}") + print(f"CUDA version: {cuda_version}") +elif device == "mps": + print(f"Is MPS (Metal Performance Shader) built? {mps_built}") + print(f"Is MPS available? {mps_available}") + +# Create data and send it to the device +x = torch.rand(size=(3, 4)).to(device) +print(x) diff --git a/src/backend/go.mod b/src/backend/go.mod new file mode 100644 index 00000000..446d5ff3 --- /dev/null +++ b/src/backend/go.mod @@ -0,0 +1,139 @@ +module cognix.ch/api/v2 + +go 1.22.1 + +require ( + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 + github.com/caarlos0/env/v10 v10.0.0 + github.com/deluan/flowllm v0.0.0-20230502144710-1414e4b4985b + github.com/gabriel-vasile/mimetype v1.4.3 + github.com/gin-contrib/cors v1.7.0 + github.com/gin-gonic/gin v1.9.1 + github.com/go-co-op/gocron/v2 v2.5.0 + github.com/go-ozzo/ozzo-validation/v4 v4.3.0 + github.com/go-pg/pg/v10 v10.12.0 + github.com/go-resty/resty/v2 v2.12.0 + github.com/gocolly/colly/v2 v2.1.0 + github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang/protobuf v1.5.4 + github.com/google/uuid v1.6.0 + github.com/lib/pq v1.10.9 + github.com/milvus-io/milvus-sdk-go/v2 v2.3.6 + github.com/minio/minio-go/v7 v7.0.69 + github.com/nats-io/nats.go v1.34.1 + github.com/sashabaranov/go-openai v1.20.4 + github.com/shopspring/decimal v1.4.0 + github.com/stretchr/testify v1.9.0 + github.com/swaggo/files v1.0.1 + github.com/swaggo/gin-swagger v1.6.0 + github.com/swaggo/swag v1.16.3 + github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.4 + go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/fx v1.21.0 + go.uber.org/zap v1.27.0 + golang.org/x/net v0.24.0 + golang.org/x/oauth2 v0.19.0 + google.golang.org/grpc v1.63.2 + google.golang.org/protobuf v1.33.0 + jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 +) + +require ( + cloud.google.com/go/compute v1.24.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/PuerkitoBio/goquery v1.5.1 // indirect + github.com/andybalholm/cascadia v1.2.0 // indirect + github.com/antchfx/htmlquery v1.2.3 // indirect + github.com/antchfx/xmlquery v1.2.4 // indirect + github.com/antchfx/xpath v1.1.8 // indirect + github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect + github.com/bytedance/sonic v1.11.2 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.1 // indirect + github.com/cockroachdb/errors v1.9.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect + github.com/cockroachdb/redact v1.1.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/getsentry/sentry-go v0.12.0 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-pg/zerochecker v0.2.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/milvus-io/milvus-proto/go-api/v2 v2.3.5 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/ginkgo/v2 v2.15.0 // indirect + github.com/onsi/gomega v1.31.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect + github.com/temoto/robotstxt v1.1.1 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.4 // indirect + github.com/vmihailenco/bufpool v0.1.11 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect + github.com/vmihailenco/tagparser v0.1.2 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.uber.org/dig v1.17.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.7.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + mellium.im/sasl v0.3.1 // indirect +) diff --git a/src/backend/go.sum b/src/backend/go.sum new file mode 100644 index 00000000..67d3296c --- /dev/null +++ b/src/backend/go.sum @@ -0,0 +1,710 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= +github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= +github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= +github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= +github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= +github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= +github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= +github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= +github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= +github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= +github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deluan/flowllm v0.0.0-20230502144710-1414e4b4985b h1:LJYffjwlkEvIQH+E4aw2gEZzyTEDPRHC7eE1piLVBuw= +github.com/deluan/flowllm v0.0.0-20230502144710-1414e4b4985b/go.mod h1:Hrk6DnRn3dg1p2F4GMKCPh0v4kaxat4mU3ZD0/L0MdM= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/gin-contrib/cors v1.7.0 h1:wZX2wuZ0o7rV2/1i7gb4Jn+gW7HBqaP91fizJkBUJOA= +github.com/gin-contrib/cors v1.7.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= +github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= +github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-co-op/gocron/v2 v2.5.0 h1:ff/TJX9GdTJBDL1il9cyd/Sj3WnS+BB7ZzwHKSNL5p8= +github.com/go-co-op/gocron/v2 v2.5.0/go.mod h1:ckPQw96ZuZLRUGu88vVpd9a6d9HakI14KWahFZtGvNw= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-faker/faker/v4 v4.1.0 h1:ffuWmpDrducIUOO0QSKSF5Q2dxAht+dhsT9FvVHhPEI= +github.com/go-faker/faker/v4 v4.1.0/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= +github.com/go-pg/pg/v10 v10.12.0 h1:rBmfDDHTN7FQW0OemYmcn5UuBy6wkYWgh/Oqt1OBEB8= +github.com/go-pg/pg/v10 v10.12.0/go.mod h1:USA08CdIasAn0F6wC1nBf5nQhMHewVQodWoH89RPXaI= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= +github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= +github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= +github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20220829040838-70bd9ae97f40 h1:ykKxL12NZd3JmWZnyqarJGsF73M9Xhtrik/FEtEeFRE= +github.com/google/pprof v0.0.0-20220829040838-70bd9ae97f40/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= +github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= +github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= +github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= +github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/milvus-io/milvus-proto/go-api/v2 v2.3.5 h1:4XDy6ATB2Z0fl4Jn0hS6BT6/8YaE0d+ZUf4uBH+Z0Do= +github.com/milvus-io/milvus-proto/go-api/v2 v2.3.5/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= +github.com/milvus-io/milvus-sdk-go/v2 v2.3.6 h1:JVn9OdaronLGmtpxvamQf523mtn3Z/CRxkSZCMWutV4= +github.com/milvus-io/milvus-sdk-go/v2 v2.3.6/go.mod h1:bYFSXVxEj6A/T8BfiR+xkofKbAVZpWiDvKr3SzYUWiA= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0= +github.com/minio/minio-go/v7 v7.0.69/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/sashabaranov/go-openai v1.20.4 h1:095xQ/fAtRa0+Rj21sezVJABgKfGPNbyx/sAN/hJUmg= +github.com/sashabaranov/go-openai v1.20.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= +github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= +github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= +github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= +github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= +github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.4 h1:A6+6ZGgLRoUTD+Jkw/Ph0g8HKiHUsiGlbngcSqBaHsw= +github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.4/go.mod h1:gNYQe4RRVyszriFOhuMpwpAu4kdoFlZgcsw6dcIDFWg= +github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.4 h1:/4mU8NB88+6u9JVKlkdD6HjrhRM1V1KRTsJaU8FSr8I= +github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.4/go.mod h1:JoL6Kg6zYo9WtK5Y715GWItSUNpWprRYj5wgO01h00g= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 h1:1f31+6grJmV3X4lxcEvUy13i5/kfDw1nJZwhd8mA4tg= +go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0/go.mod h1:1P/02zM3OwkX9uki+Wmxw3a5GVb6KUXRsa7m7bOC9Fg= +go.opentelemetry.io/contrib/propagators/b3 v1.24.0 h1:n4xwCdTx3pZqZs2CjS/CUZAs03y3dZcGhC/FepKtEUY= +go.opentelemetry.io/contrib/propagators/b3 v1.24.0/go.mod h1:k5wRxKRU2uXx2F8uNJ4TaonuEO/V7/5xoz7kdsDACT8= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= +go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.21.0 h1:qqD6k7PyFHONffW5speYx403ywanuASqU4Rqdpc22XY= +go.uber.org/fx v1.21.0/go.mod h1:HT2M7d7RHo+ebKGh9NRcrsrHHfpZ60nW3QRubMRfv48= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 h1:Xs9lu+tLXxLIfuci70nG4cpwaRC+mRQPUL7LoIeDJC4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= +google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 h1:6YFJoB+0fUH6X3xU/G2tQqCYg+PkGtnZ5nMR5rpw72g= +jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/src/backend/lib_py/.DS_Store b/src/backend/lib_py/.DS_Store new file mode 100644 index 00000000..e1c970a4 Binary files /dev/null and b/src/backend/lib_py/.DS_Store differ diff --git a/src/backend/lib_py/__init__.py b/src/backend/lib_py/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/lib_py/sentence_encoder.py b/src/backend/lib_py/sentence_encoder.py new file mode 100644 index 00000000..36370fc4 --- /dev/null +++ b/src/backend/lib_py/sentence_encoder.py @@ -0,0 +1,36 @@ +import os + +# todo: change local path to S3 storage +class SentenceEncoder: + def __init__(self, model_name, local_model_dir='models'): + """ + Initializes an instance of SentenceEncoder, attempting to load the model from a local directory first. + If the model is not available locally, it downloads from Hugging Face and saves it locally. + + Parameters: + model_name (str): The name of the model to load or download. + local_model_dir (str): The directory to check for the model and to save the model. + """ + self.model_path = os.path.join(local_model_dir, model_name) + + + + def embed(self, text): + """ + Encodes the provided text using the loaded SentenceTransformer model. + + Parameters: + text (str): The text to be encoded. + + Returns: + list: A list of floats representing the encoded text. + """ + # Use the loaded model to encode the text + return "" + +# # Example usage +# if __name__ == "__main__": +# model_name = 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2' +# encoder = TextEncoder(model_name) # Create an instance of TextEncoder with a specific model +# encoded_data = encoder.embed("explain routed events in WPF") # Call the embed method with a sample text +# print(encoded_data) # Print the encoded data diff --git a/src/backend/lib_py/telemetry.py b/src/backend/lib_py/telemetry.py new file mode 100644 index 00000000..5f347f9f --- /dev/null +++ b/src/backend/lib_py/telemetry.py @@ -0,0 +1,34 @@ +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.grpc import GrpcInstrumentorServer +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.instrumentation.grpc import server_interceptor + +class OpenTelemetryManager: + def __init__(self): + self.trace_provider = TracerProvider() + trace.set_tracer_provider(self.trace_provider) + self.tracer = trace.get_tracer(__name__) + + otlp_exporter = OTLPSpanExporter() + self.trace_provider.add_span_processor(BatchSpanProcessor(otlp_exporter)) + + self.meter_provider = MeterProvider() + metric_exporter = OTLPMetricExporter() + metric_reader = PeriodicExportingMetricReader(metric_exporter) + self.meter_provider._all_metric_readers = metric_reader + self.meter = self.meter_provider.get_meter(__name__) + + self.embedding_time_metric = self.meter.create_histogram("embedding_creation_time", description="Time taken to create embeddings") + + GrpcInstrumentorServer().instrument() + + def start_trace(self, span_name): + return self.tracer.start_as_current_span(span_name) + + def record_metric(self, start_time, end_time): + self.embedding_time_metric.record(end_time - start_time) \ No newline at end of file diff --git a/src/backend/loki-config.yaml b/src/backend/loki-config.yaml new file mode 100644 index 00000000..e448dfd9 --- /dev/null +++ b/src/backend/loki-config.yaml @@ -0,0 +1,50 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false diff --git a/src/backend/migration/Dockerfile b/src/backend/migration/Dockerfile new file mode 100644 index 00000000..96aeef21 --- /dev/null +++ b/src/backend/migration/Dockerfile @@ -0,0 +1,12 @@ +# select image +FROM golang:1.22.1-alpine + +# install dependensies and migration tool +RUN apk add --no-cache build-base git && go install github.com/pressly/goose/cmd/goose@latest + +# copy migration files. +COPY versions /versions + +# run migration +WORKDIR /versions +CMD goose postgres "${COCKROACH_CLIENT_DATABASE_URL_GOLANG}" up diff --git a/src/backend/migration/migration-service-deployment.yaml b/src/backend/migration/migration-service-deployment.yaml new file mode 100644 index 00000000..ce74b175 --- /dev/null +++ b/src/backend/migration/migration-service-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: migrationservice +spec: + replicas: 1 + selector: + matchLabels: + app: migrationservice + template: + metadata: + labels: + app: migrationservice + spec: + containers: + - name: migrationservice + image: cognixacr.azurecr.io/cognix/migrationservice:main + envFrom: + - configMapRef: + name: cockroach-cli + volumeMounts: + - mountPath: /versions + name: storage-volume + imagePullSecrets: + - name: pullsecret + volumes: + - name: storage-volume + persistentVolumeClaim: + claimName: versions-volume +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: versions-volume + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: azurefile-csi \ No newline at end of file diff --git a/src/backend/migration/versions/20240322095219_init_database.sql b/src/backend/migration/versions/20240322095219_init_database.sql new file mode 100644 index 00000000..ff8fd385 --- /dev/null +++ b/src/backend/migration/versions/20240322095219_init_database.sql @@ -0,0 +1,206 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS tenants ( + id uuid PRIMARY KEY, + name varchar(255), + configuration jsonb not null default '{}'::jsonb +); +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + tenant_id uuid NOT NULL REFERENCES tenants(id), + user_name varchar(255) UNIQUE NOT NULL, + first_name varchar(255), + last_name varchar(255), + external_id text, + roles text[] NOT NULL DEFAULT '{}'::text[] +); +CREATE TABLE IF NOT EXISTS llm ( + id SERIAL PRIMARY KEY, + tenant_id uuid NOT NULL REFERENCES tenants(id), + name varchar(255) NOT NULL, + model_id varchar(255) NOT NULL, + url varchar NOT NULL, + api_key varchar, + endpoint varchar, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT now(), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +CREATE TABLE IF NOT EXISTS embedding_models ( + id SERIAL PRIMARY KEY, + tenant_id uuid NOT NULL, + model_id varchar NOT NULL, + model_name varchar NOT NULL, + model_dim integer NOT NULL, + normalize boolean NOT NULL, + query_prefix varchar NOT NULL, + passage_prefix varchar NOT NULL, + index_name varchar NOT NULL, + "url" varchar, + is_active boolean NOT NULL DEFAULT false, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT now(), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +CREATE TABLE IF NOT EXISTS personas ( + id SERIAL PRIMARY KEY, + name varchar NOT NULL, + llm_id bigint references llm(id), + default_persona boolean NOT NULL, + description varchar NOT NULL, + tenant_id uuid NOT NULL references tenants(id), + is_visible boolean NOT NULL, + display_priority integer, + starter_messages jsonb NOT NULL DEFAULT '{}'::jsonb, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT now(), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +CREATE TABLE IF NOT EXISTS prompts ( + id SERIAL PRIMARY KEY, + persona_id bigint NOT NULL REFERENCES personas(id), + user_id uuid NOT NULL references users(id), + name varchar NOT NULL, + description varchar NOT NULL, + system_prompt text NOT NULL, + task_prompt text NOT NULL, + include_citations boolean NOT NULL, + datetime_aware boolean NOT NULL, + default_prompt boolean NOT NULL, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT (now()), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +CREATE TABLE IF NOT EXISTS credentials ( + id SERIAL PRIMARY KEY, + credential_json jsonb NOT NULL DEFAULT '{}'::jsonb, + user_id uuid NOT NULL references users(id), + tenant_id uuid NOT NULL references tenants(id), + source varchar(50) NOT NULL, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT (now()), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE, + shared boolean NOT NULL +); + + +CREATE TABLE IF NOT EXISTS connectors ( + id SERIAL PRIMARY KEY, + credential_id bigint NULL references credentials(id), + name varchar NOT NULL, + source varchar(50) NOT NULL, + input_type varchar(10), + connector_specific_config jsonb NOT NULL, + refresh_freq integer, + user_id uuid NOT NULL references users(id), + tenant_id uuid NOT NULL references tenants(id), + shared boolean NOT NULL, + disabled boolean NOT NULL, + last_successful_index_time timestamp WITHOUT TIME ZONE, + last_attempt_status varchar, + total_docs_indexed integer NOT NULL, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT (now()), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +CREATE TABLE IF NOT EXISTS chat_sessions ( + id SERIAL PRIMARY KEY, + user_id uuid NOT NULL references users(id), + description text NOT NULL, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT (now()), + deleted_date timestamp WITHOUT TIME ZONE, + persona_id integer NOT NULL references personas(id), + one_shot boolean NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_messages ( + id SERIAL PRIMARY KEY, + chat_session_id bigint NOT NULL references chat_sessions(id), + message text NOT NULL, + message_type varchar(9) NOT NULL, + time_sent timestamp WITHOUT TIME ZONE NOT NULL DEFAULT (now()), + token_count integer NOT NULL, + parent_message integer, + latest_child_message integer, + rephrased_query text, + citations jsonb NOT NULL default '{}'::jsonb, + error text +); + +CREATE TABLE chat_message_feedbacks ( + id SERIAL NOT NULL PRIMARY KEY, + chat_message_id BIGINT NOT NULL REFERENCES chat_messages, + user_id UUID NOT NULL REFERENCES users, + up_votes BOOLEAN NOT NULL, + feedback VARCHAR NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS documents ( + id serial PRIMARY KEY NOT NULL, + document_id varchar NOT NULL , + connector_id bigint NOT NULL references connectors(id), + boost integer NOT NULL, + hidden boolean NOT NULL, + semantic_id varchar NOT NULL, + link varchar, + from_ingestion_api boolean, + signature text, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT now(), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +create table document_feedbacks( + id serial primary key, + document_id bigint references documents(id) , + user_id uuid references users(id), + document_rank integer not null, + up_votes boolean not null, + feedback varchar not null default '' +); + + +CREATE TABLE IF NOT EXISTS document_sets ( + id SERIAL PRIMARY KEY, + user_id uuid references users(id), + name varchar NOT NULL, + description varchar NOT NULL, + is_up_to_date boolean NOT NULL, + created_date timestamp WITHOUT TIME ZONE NOT NULL DEFAULT (now()), + updated_date timestamp WITHOUT TIME ZONE, + deleted_date timestamp WITHOUT TIME ZONE +); + +CREATE TABLE IF NOT EXISTS document_set_connector_pairs ( + document_set_id integer NOT NULL references document_sets(id), + connector_id integer NOT NULL references connectors(id), + is_current boolean NOT NULL, + primary key (document_set_id,connector_id) +); + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS embedding_models; +DROP TABLE IF EXISTS chat_message_feedback; +DROP TABLE IF EXISTS chat_messages; +DROP TABLE IF EXISTS chat_sessions; +DROP TABLE IF EXISTS prompts; +DROP TABLE IF EXISTS personas; +DROP TABLE IF EXISTS llm; +DROP TABLE IF EXISTS document_feedbacks; +DROP TABLE IF EXISTS documents; +DROP TABLE IF EXISTS document_set_connector_pairs ; +DROP TABLE IF EXISTS document_sets; +DROP TABLE IF EXISTS connectors; +DROP TABLE IF EXISTS credentials; +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS tenants; + +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240426103932_test.sql b/src/backend/migration/versions/20240426103932_test.sql new file mode 100644 index 00000000..04f4033d --- /dev/null +++ b/src/backend/migration/versions/20240426103932_test.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS test ( + id uuid PRIMARY KEY, + name text + +); +DROP TABLE test ; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query'; + +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240506092247_alter_document.sql b/src/backend/migration/versions/20240506092247_alter_document.sql new file mode 100644 index 00000000..f10a459e --- /dev/null +++ b/src/backend/migration/versions/20240506092247_alter_document.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE documents DROP COLUMN IF EXISTS boost; +ALTER TABLE documents DROP COLUMN IF EXISTS hidden; +ALTER TABLE documents DROP COLUMN IF EXISTS semantic_id; +ALTER TABLE documents DROP COLUMN IF EXISTS from_ingestion_api; +ALTER TABLE connectors DROP COLUMN IF EXISTS input_type; +ALTER TABLE embedding_models DROP COLUMN IF EXISTS normalize; +ALTER TABLE embedding_models DROP COLUMN IF EXISTS query_prefix; +ALTER TABLE embedding_models DROP COLUMN IF EXISTS passage_prefix; +ALTER TABLE embedding_models DROP COLUMN IF EXISTS index_name; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240530143908_redesign_database.sql b/src/backend/migration/versions/20240530143908_redesign_database.sql new file mode 100644 index 00000000..42568468 --- /dev/null +++ b/src/backend/migration/versions/20240530143908_redesign_database.sql @@ -0,0 +1,64 @@ +-- +goose Up +-- +goose StatementBegin +DROP TABLE IF EXISTS document_feedbacks; +DROP TABLE IF EXISTS document_set_connector_pairs; +DROP TABLE IF EXISTS document_sets; +DROP TABLE IF EXISTS documents; + +CREATE TABLE documents ( + id SERIAL PRIMARY KEY NOT NULL, + parent_id bigint REFERENCES documents(id), -- Allows nulls, used for URLs + connector_id bigint NOT NULL REFERENCES connectors(id), + source_id text NOT NULL, -- unique id from source url for web, id for other services + url text, -- url for web connector, link (minio:bucket:file) for file in minio + signature text, + chunking_session uuid, -- Allows nulls + analyzed bool NOT NULL DEFAULT FALSE, -- default false, true when semantic created the embeddings in the vector db + creation_date timestamp WITHOUT TIME ZONE NOT NULL, --datetime utc IMPORTANT now() will not get the utc date!!!! + last_update timestamp WITHOUT TIME ZONE +); + +ALTER TABLE connectors RENAME COLUMN source TO type; +ALTER TABLE connectors RENAME COLUMN created_date TO creation_date; +ALTER TABLE connectors ALTER COLUMN creation_date DROP DEFAULT; +ALTER TABLE connectors RENAME COLUMN updated_date TO last_update; +ALTER TABLE connectors RENAME COLUMN last_successful_index_time TO last_successful_index_date; +ALTER TABLE connectors ALTER COLUMN tenant_id DROP NOT NULL; +ALTER TABLE connectors DROP COLUMN IF EXISTS shared; + +ALTER TABLE chat_messages ALTER COLUMN time_sent DROP DEFAULT; + +ALTER TABLE chat_sessions RENAME COLUMN created_date TO creation_date; +ALTER TABLE chat_sessions ALTER COLUMN creation_date DROP DEFAULT; + +ALTER TABLE credentials RENAME COLUMN created_date TO creation_date; +ALTER TABLE credentials ALTER COLUMN creation_date DROP DEFAULT; +ALTER TABLE credentials RENAME COLUMN updated_date TO last_update; + +ALTER TABLE embedding_models RENAME COLUMN created_date TO creation_date; +ALTER TABLE embedding_models ALTER COLUMN creation_date DROP DEFAULT; +ALTER TABLE embedding_models RENAME COLUMN updated_date TO last_update; + +ALTER TABLE llm RENAME COLUMN created_date TO creation_date; +ALTER TABLE llm ALTER COLUMN creation_date DROP DEFAULT; +ALTER TABLE llm RENAME COLUMN updated_date TO last_update; + +ALTER TABLE personas RENAME COLUMN created_date TO creation_date; +ALTER TABLE personas ALTER COLUMN creation_date DROP DEFAULT; +ALTER TABLE personas RENAME COLUMN updated_date TO last_update; + +ALTER TABLE prompts RENAME COLUMN created_date TO creation_date; +ALTER TABLE prompts ALTER COLUMN creation_date DROP DEFAULT; +ALTER TABLE prompts RENAME COLUMN updated_date TO last_update; + +ALTER TABLE prompts DROP COLUMN IF EXISTS include_citations; +ALTER TABLE prompts DROP COLUMN IF EXISTS default_prompt; +ALTER TABLE prompts DROP COLUMN IF EXISTS datetime_aware; + +ALTER TABLE llm RENAME TO llms; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query'; +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240604090037_remove_credential_table.sql b/src/backend/migration/versions/20240604090037_remove_credential_table.sql new file mode 100644 index 00000000..b7953521 --- /dev/null +++ b/src/backend/migration/versions/20240604090037_remove_credential_table.sql @@ -0,0 +1,30 @@ +-- +goose Up +-- +goose StatementBegin +alter table connectors drop column if exists credential_id; +alter table connectors drop column if exists disabled; +alter table connectors rename column last_successful_index_date to last_successful_analyzed; +alter table connectors rename column total_docs_indexed to total_docs_analyzed; +alter table connectors rename column last_attempt_status to status; +drop table if exists credentials; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +create table if not exists credentials +( + id bigint default unique_rowid() not null primary key, + credential_json jsonb default '{}'::JSONB not null, + user_id uuid not null references public.users, + tenant_id uuid not null references public.tenants, + source varchar(50) not null, + creation_date timestamp not null, + last_update timestamp, + deleted_date timestamp, + shared boolean not null +); +alter table connectors add column if not exists credential_id bigint references credentials(id); +alter table connectors rename column status to last_attempt_status; +alter table connectors add column if not exists disabled bool ; +alter table connectors rename column last_successful_analyzed to last_successful_index_date; +alter table connectors rename column total_docs_analyzed to total_docs_indexed; +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240604132415_add_message_document_pairs.sql b/src/backend/migration/versions/20240604132415_add_message_document_pairs.sql new file mode 100644 index 00000000..c5bf04b5 --- /dev/null +++ b/src/backend/migration/versions/20240604132415_add_message_document_pairs.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS chat_message_document_pairs( + id SERIAL PRIMARY KEY, + chat_message_id bigint NOT NULL REFERENCES chat_messages(id) ON DELETE CASCADE, + document_id bigint NOT NULL REFERENCES documents(id) ON DELETE CASCADE +); +ALTER TABLE chat_messages DROP COLUMN IF EXISTS citations; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROp TABLE IF EXISTS chat_message_document_pairs; +ALTER TABLE chat_messages ADD COLUMN IF NOT EXISTS citations jsonb; +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240617132246_add_connector_state.sql b/src/backend/migration/versions/20240617132246_add_connector_state.sql new file mode 100644 index 00000000..e7b2615c --- /dev/null +++ b/src/backend/migration/versions/20240617132246_add_connector_state.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE connectors ADD COLUMN IF NOT EXISTS state jsonb not null default '{}'::jsonb; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE connectors DROP COLUMN IF EXISTS state; +-- +goose StatementEnd diff --git a/src/backend/migration/versions/20240624092512_add_original_url.sql b/src/backend/migration/versions/20240624092512_add_original_url.sql new file mode 100644 index 00000000..31dcdcc3 --- /dev/null +++ b/src/backend/migration/versions/20240624092512_add_original_url.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE documents ADD COLUMN IF NOT EXISTS original_url text not null default ''; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE documents DROP COLUMN IF EXISTS original_url; +-- +goose StatementEnd diff --git a/src/backend/migration/versions/README.md b/src/backend/migration/versions/README.md new file mode 100644 index 00000000..7d41ab45 --- /dev/null +++ b/src/backend/migration/versions/README.md @@ -0,0 +1,9 @@ +## Migrations + +Migrations are handled by [goose](https://github.com/pressly/goose) + +To install goose, run the following: +```sh +$ go get -u github.com/pressly/goose/cmd/goose +``` +`docker-compose run --rm migration /bin/sh` \ No newline at end of file diff --git a/src/backend/orchestrator/main.go b/src/backend/orchestrator/main.go new file mode 100644 index 00000000..f3a41af4 --- /dev/null +++ b/src/backend/orchestrator/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +func main() { + utils.InitLogger(true) + app := fx.New(Module) + + app.Run() +} diff --git a/src/backend/orchestrator/mocks/connector_repo_mock.go b/src/backend/orchestrator/mocks/connector_repo_mock.go new file mode 100644 index 00000000..0d70109e --- /dev/null +++ b/src/backend/orchestrator/mocks/connector_repo_mock.go @@ -0,0 +1,163 @@ +package mocks + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/repository" + "context" + "github.com/google/uuid" + "github.com/shopspring/decimal" + "go.uber.org/zap" + "time" +) + +var MockedConnectors = map[int64]*model.Connector{ + 1: { + ID: decimal.NewFromInt(1), + Name: "web connector ready to process", + Type: model.SourceTypeWEB, + ConnectorSpecificConfig: model.JSONMap{ + "url": "http://testurl", + }, + RefreshFreq: 60, + Status: model.ConnectorStatusReadyToProcessed, + TotalDocsAnalyzed: 0, + CreationDate: time.Now().UTC(), + User: &model.User{ + EmbeddingModel: &model.EmbeddingModel{ + ModelID: "", + ModelDim: 3, + }, + }, + }, + 2: { + ID: decimal.NewFromInt(2), + Name: "File connector ready to process", + Type: model.SourceTypeFile, + ConnectorSpecificConfig: model.JSONMap{ + "file_name": "file name", + }, + RefreshFreq: 60, + Status: model.ConnectorStatusReadyToProcessed, + TotalDocsAnalyzed: 0, + CreationDate: time.Now().UTC(), + User: &model.User{ + EmbeddingModel: &model.EmbeddingModel{ + ModelID: "", + ModelDim: 3, + }, + }, + }, + 3: { + ID: decimal.NewFromInt(3), + Name: "One drive connector disabled", + Type: model.SourceTypeOneDrive, + ConnectorSpecificConfig: model.JSONMap{ + "file_name": "file name", + }, + RefreshFreq: 60, + Status: model.ConnectorStatusDisabled, + TotalDocsAnalyzed: 0, + CreationDate: time.Now().UTC(), + User: &model.User{ + EmbeddingModel: &model.EmbeddingModel{ + ModelID: "", + ModelDim: 3, + }, + }, + }, + 4: { + ID: decimal.NewFromInt(2), + Name: "File connector without embedding model", + Type: model.SourceTypeFile, + ConnectorSpecificConfig: model.JSONMap{ + "file_name": "file name", + }, + RefreshFreq: 60, + Status: model.ConnectorStatusReadyToProcessed, + TotalDocsAnalyzed: 0, + CreationDate: time.Now().UTC(), + User: &model.User{}, + }, + 5: { + ID: decimal.NewFromInt(5), + Name: "One drive connector ready for processing ", + Type: model.SourceTypeOneDrive, + ConnectorSpecificConfig: model.JSONMap{ + "file_name": "file name", + }, + RefreshFreq: 60, + Status: model.ConnectorStatusReadyToProcessed, + TotalDocsAnalyzed: 0, + CreationDate: time.Now().UTC(), + User: &model.User{ + EmbeddingModel: &model.EmbeddingModel{ + ModelID: "", + ModelDim: 3, + }, + }, + }, +} + +type MockConnectorRepo struct { + workCh chan int + expectedIteration int + iteration int +} + +func (m *MockConnectorRepo) GetActive(ctx context.Context) ([]*model.Connector, error) { + result := make([]*model.Connector, 0) + zap.S().Errorf("load iteration %d", m.iteration) + if m.iteration >= m.expectedIteration { + close(m.workCh) + return result, nil + } + m.iteration++ + zap.S().Info("connector for running ") + for _, conn := range MockedConnectors { + if conn.Status == model.ConnectorStatusReadyToProcessed || + conn.Status == model.ConnectorStatusSuccess || + conn.Status == model.ConnectorStatusError { + zap.S().Infof("| %s \t\t| %s \t\t | %s \t\t | %v |", + conn.Name, conn.Type, conn.Status, conn.LastUpdate) + result = append(result, conn) + } + } + return result, nil +} + +func (m *MockConnectorRepo) GetAllByUser(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockConnectorRepo) GetByIDAndUser(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockConnectorRepo) GetByID(ctx context.Context, id int64) (*model.Connector, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockConnectorRepo) GetBySource(ctx context.Context, tenantID, userID uuid.UUID, source model.SourceType) (*model.Connector, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockConnectorRepo) Create(ctx context.Context, connector *model.Connector) error { + //TODO implement me + panic("implement me") +} + +func (m *MockConnectorRepo) Update(ctx context.Context, connector *model.Connector) error { + MockedConnectors[connector.ID.IntPart()] = connector + return nil +} + +func NewMockConnectorRepo(iteration int, workCh chan int) repository.ConnectorRepository { + return &MockConnectorRepo{ + workCh: workCh, + expectedIteration: iteration, + } +} diff --git a/src/backend/orchestrator/mocks/document_repo_mock.go b/src/backend/orchestrator/mocks/document_repo_mock.go new file mode 100644 index 00000000..4c070e6c --- /dev/null +++ b/src/backend/orchestrator/mocks/document_repo_mock.go @@ -0,0 +1,43 @@ +package mocks + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/repository" + "context" + "github.com/shopspring/decimal" +) + +type MockDocumentRepository struct{} + +func (m MockDocumentRepository) FindByConnectorIDAndUser(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) { + //TODO implement me + panic("implement me") +} + +func (m MockDocumentRepository) FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) { + //TODO implement me + panic("implement me") +} + +func (m MockDocumentRepository) FindByID(ctx context.Context, id int64) (*model.Document, error) { + //TODO implement me + panic("implement me") +} + +func (m MockDocumentRepository) Create(ctx context.Context, document *model.Document) error { + document.ID = decimal.NewFromInt(1) + return nil +} + +func (m MockDocumentRepository) Update(ctx context.Context, document *model.Document) error { + return nil +} + +func (m MockDocumentRepository) DeleteByIDS(ctx context.Context, ids ...int64) error { + //TODO implement me + panic("implement me") +} + +func NewMockDocumentRepo() repository.DocumentRepository { + return &MockDocumentRepository{} +} diff --git a/src/backend/orchestrator/mocks/messenger_mock.go b/src/backend/orchestrator/mocks/messenger_mock.go new file mode 100644 index 00000000..5eaf1ded --- /dev/null +++ b/src/backend/orchestrator/mocks/messenger_mock.go @@ -0,0 +1,72 @@ +package mocks + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "github.com/go-pg/pg/v10" + proto2 "github.com/golang/protobuf/proto" + "go.uber.org/zap" + "time" +) + +type MockMessenger struct { + workCh chan int +} + +func (m MockMessenger) Publish(ctx context.Context, streamName, topic string, body proto2.Message) error { + + if streamName == "semantic" { + semantic := body.(*proto.SemanticData) + conn := MockedConnectors[semantic.ConnectorId] + zap.S().Infof("befor sending to semantic .... ") + zap.S().Infof("| %s \t\t| %s \t\t | %s \t\t | %v |", + conn.Name, conn.Type, conn.Status, conn.LastUpdate) + + conn.Status = model.ConnectorStatusSuccess + conn.LastUpdate = pg.NullTime{time.Now().UTC()} + zap.S().Infof("emulate semantic work .... ") + zap.S().Infof("| %s \t\t| %s \t\t | %s \t\t | %v |", + conn.Name, conn.Type, conn.Status, conn.LastUpdate) + return nil + } + if streamName == "connector" { + connRequest := body.(*proto.ConnectorRequest) + conn := MockedConnectors[connRequest.Id] + zap.S().Infof("befor sending to connector .... ") + zap.S().Infof("| %s \t\t| %s \t\t | %s \t\t | %v |", + conn.Name, conn.Type, conn.Status, conn.LastUpdate) + + conn.Status = model.ConnectorStatusWorking + conn.LastUpdate = pg.NullTime{time.Now().UTC()} + zap.S().Infof("emulate connector work .... ") + zap.S().Infof("| %s \t\t| %s \t\t | %s \t\t | %v |", + conn.Name, conn.Type, conn.Status, conn.LastUpdate) + return nil + } + return nil +} + +func (m MockMessenger) Listen(ctx context.Context, streamName, topic string, handler messaging.MessageHandler) error { + + return nil +} + +func (m MockMessenger) StreamConfig() *messaging.StreamConfig { + return &messaging.StreamConfig{ + ConnectorStreamName: "connector", + SemanticStreamName: "semantic", + } +} + +func (m MockMessenger) Close() { + //TODO implement me + panic("implement me") +} + +func NewMockMessenger(workCh chan int) messaging.Client { + return &MockMessenger{ + workCh: workCh, + } +} diff --git a/src/backend/orchestrator/module.go b/src/backend/orchestrator/module.go new file mode 100644 index 00000000..f1c1b952 --- /dev/null +++ b/src/backend/orchestrator/module.go @@ -0,0 +1,45 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/utils" + "context" + "go.uber.org/fx" +) + +type Config struct { + OAuthURL string `env:"OAUTH_URL,required"` + RenewInterval int `env:"ORCHESTRATOR_RENEW_INTERVAL" envDefault:"30"` + FileSizeLimit int `env:"FILE_SIZE_LIMIT,required"` +} + +var Module = fx.Options( + repository.DatabaseModule, + messaging.NatsModule, + fx.Provide( + func() (*Config, error) { + cfg := Config{} + err := utils.ReadConfig(&cfg) + return &cfg, err + }, + repository.NewConnectorRepository, + repository.NewDocumentRepository, + NewServer, + ), + fx.Invoke(RunServer), +) + +func RunServer(lc fx.Lifecycle, server *Server) error { + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return server.run(ctx) + }, + OnStop: func(ctx context.Context) error { + + return nil + }, + }) + return nil +} diff --git a/src/backend/orchestrator/server.go b/src/backend/orchestrator/server.go new file mode 100644 index 00000000..5335d9a2 --- /dev/null +++ b/src/backend/orchestrator/server.go @@ -0,0 +1,84 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/repository" + "context" + "github.com/go-co-op/gocron/v2" + "go.uber.org/zap" + "time" +) + +type Server struct { + renewInterval time.Duration + connectorRepo repository.ConnectorRepository + docRepo repository.DocumentRepository + messenger messaging.Client + scheduler gocron.Scheduler + streamCfg *messaging.StreamConfig + cfg *Config +} + +func NewServer( + cfg *Config, + connectorRepo repository.ConnectorRepository, + docRepo repository.DocumentRepository, + messenger messaging.Client, + messagingCfg *messaging.Config) (*Server, error) { + s, err := gocron.NewScheduler() + if err != nil { + return nil, err + } + + return &Server{connectorRepo: connectorRepo, + docRepo: docRepo, + renewInterval: time.Duration(cfg.RenewInterval) * time.Second, + cfg: cfg, + messenger: messenger, + streamCfg: messagingCfg.Stream, + scheduler: s, + }, nil +} + +func (s *Server) run(ctx context.Context) error { + zap.S().Infof("Schedule reload task") + go s.schedule() + zap.S().Infof("Start listener ...") + go s.loadFromDatabase() + return nil +} + +// loadFromDatabase load connectors from database and run if needed +func (s *Server) loadFromDatabase() error { + ctx := context.Background() + if !s.messenger.IsOnline() { + zap.S().Infof("Messenger is offline.") + return nil + } + zap.S().Infof("Loading connectors from db") + connectors, err := s.connectorRepo.GetActive(ctx) + if err != nil { + zap.S().Errorf("Load connectors failed: %v", err) + return err + } + for _, connector := range connectors { + if err = NewTrigger(s.messenger, s.connectorRepo, s.docRepo, connector, s.cfg.FileSizeLimit, s.cfg.OAuthURL).Do(ctx); err != nil { + zap.S().Errorf("run connector %d failed: %v", connector.ID, err) + } + } + return nil +} + +func (s *Server) schedule() error { + _, err := s.scheduler.NewJob( + gocron.DurationJob(s.renewInterval), + gocron.NewTask(s.loadFromDatabase), + gocron.WithName("reload from database"), + ) + if err != nil { + return err + } + s.scheduler.Start() + return nil + +} diff --git a/src/backend/orchestrator/server_test.go b/src/backend/orchestrator/server_test.go new file mode 100644 index 00000000..1d3d00fb --- /dev/null +++ b/src/backend/orchestrator/server_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/utils" + "cognix.ch/api/v2/orchestrator/mocks" + "context" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "testing" +) + +func TestOrchestrator_Scheduler(t *testing.T) { + //b := &bytes.Buffer{} + // call the constructor from your test code with the arbitrary writer + //mycore := NewCustomLogger(os.Stderr) + //zap.ReplaceGlobals(zap.New(mycore)) + utils.InitLogger(true) + workCh := make(chan int, 10) + + srv, err := NewServer( + &Config{ + RenewInterval: 15, + FileSizeLimit: 1, + }, + mocks.NewMockConnectorRepo(10, workCh), + mocks.NewMockDocumentRepo(), + mocks.NewMockMessenger(workCh), + &messaging.Config{ + Stream: &messaging.StreamConfig{ + ConnectorStreamName: "connector", + SemanticStreamName: "semantic", + }, + }, + ) + zap.S().Infof("connecto in database ") + for _, conn := range mocks.MockedConnectors { + zap.S().Infof("| %s \t\t| %s \t\t | %s \t\t | %v |", + conn.Name, conn.Type, conn.Status, conn.LastUpdate) + } + + assert.NoError(t, err) + srv.run(context.Background()) + for i := range workCh { + t.Logf("iteration %d", i) + } + //t.Log(b.String()) +} diff --git a/src/backend/orchestrator/service-deployment.yaml b/src/backend/orchestrator/service-deployment.yaml new file mode 100644 index 00000000..ebfe2826 --- /dev/null +++ b/src/backend/orchestrator/service-deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orchestratorservice +spec: + replicas: 1 + selector: + matchLabels: + app: orchestratorservice + template: + metadata: + labels: + app: orchestratorservice + spec: + containers: + - name: orchestratorservice + image: cognixacr.azurecr.io/cognix/orchestratorservice:main + envFrom: + - configMapRef: + name: orchestrator-srv + - configMapRef: + name: cockroach-cli + - configMapRef: + name: nats-cli + - configMapRef: + name: oauth-cli + - configMapRef: + name: env-configmap + imagePullPolicy: Always + volumeMounts: + - mountPath: /backend + name: storage-volume + imagePullSecrets: + - name: pullsecret + volumes: + - name: storage-volume + persistentVolumeClaim: + claimName: storage-volume +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-volume + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: azurefile-csi diff --git a/src/backend/orchestrator/trigger.go b/src/backend/orchestrator/trigger.go new file mode 100644 index 00000000..ec3688d8 --- /dev/null +++ b/src/backend/orchestrator/trigger.go @@ -0,0 +1,151 @@ +package main + +import ( + "cognix.ch/api/v2/core/connector" + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "github.com/google/uuid" + + "context" + "encoding/json" + "fmt" + "github.com/go-pg/pg/v10" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" + "time" +) + +const ( + ConnectorSchedulerSpan = "connector-scheduler" +) + +type ( + trigger struct { + messenger messaging.Client + connectorRepo repository.ConnectorRepository + docRepo repository.DocumentRepository + tracer trace.Tracer + connectorModel *model.Connector + fileSizeLimit int + oauthURL string + } +) + +func (t *trigger) Do(ctx context.Context) error { + // if connector is new or + // todo we need figure out how to use multiple orchestrators instances + // one approach could be that this method will extract top x rows from the database + // and it will book them + + if t.connectorModel.User == nil || t.connectorModel.User.EmbeddingModel == nil { + return fmt.Errorf("embedding model is not configured for %s", t.connectorModel.Name) + } + zap.S().Debugf("\n------------ %s\nlast %v refresh Freq %d \nnext %v\nnow %v\nlast+refreshFreq > now %v\n------------- ", + t.connectorModel.Name, + t.connectorModel.LastUpdate.UTC(), + t.connectorModel.RefreshFreq, + t.connectorModel.LastUpdate.UTC().Add(time.Duration(t.connectorModel.RefreshFreq)*time.Second), + time.Now().UTC(), + t.connectorModel.LastUpdate.UTC().Add(time.Duration(t.connectorModel.RefreshFreq)*time.Second).Before(time.Now().UTC())) + + if t.connectorModel.LastUpdate.IsZero() || + t.connectorModel.LastUpdate.UTC().Add(time.Duration(t.connectorModel.RefreshFreq)*time.Second).Before(time.Now().UTC()) { + ctx, span := t.tracer.Start(ctx, ConnectorSchedulerSpan) + defer span.End() + span.SetAttributes(attribute.Int64(model.SpanAttributeConnectorID, t.connectorModel.ID.IntPart())) + span.SetAttributes(attribute.String(model.SpanAttributeConnectorSource, string(t.connectorModel.Type))) + + //if err := t.updateStatus(ctx, model.ConnectorStatusPending); err != nil { + // span.RecordError(err) + // return err + //} + connWF, err := connector.New(t.connectorModel, t.connectorRepo, t.oauthURL) + if err != nil { + return err + } + sessionID := uuid.New() + if err = connWF.PrepareTask(ctx, sessionID, t); err != nil { + span.RecordError(err) + zap.S().Errorf("failed to prepare task for connector %s[%d]: %v", t.connectorModel.Name, t.connectorModel.ID.IntPart(), err) + if errr := t.updateStatus(ctx, model.ConnectorStatusUnableProcess); errr != nil { + span.RecordError(errr) + } + return err + } + } + return nil +} + +// RunSemantic send message to semantic service +func (t *trigger) RunSemantic(ctx context.Context, data *proto.SemanticData) error { + + if t.connectorModel.Type == model.SourceTypeWEB || + t.connectorModel.Type == model.SourceTypeYoutube || + t.connectorModel.Type == model.SourceTypeFile { + doc := t.connectorModel.Docs[0] + var err error + // create or update document in database + if doc.ID.IntPart() != 0 { + err = t.docRepo.Update(ctx, doc) + } else { + err = t.docRepo.Create(ctx, doc) + } + if err != nil { + zap.S().Errorf("update document failed %v", err) + return err + } + data.DocumentId = doc.ID.IntPart() + } + if err := t.updateStatus(ctx, model.ConnectorStatusPending); err != nil { + return err + } + zap.S().Infof("send message to semantic %s", t.connectorModel.Name) + buf, _ := json.Marshal(data) + zap.S().Debugf(" message payload %s", string(buf)) + return t.messenger.Publish(ctx, t.messenger.StreamConfig().SemanticStreamName, + t.messenger.StreamConfig().SemanticStreamSubject, data) +} + +// RunConnector send message to connector service +func (t *trigger) RunConnector(ctx context.Context, data *proto.ConnectorRequest) error { + data.Params[model.ParamFileLimit] = fmt.Sprintf("%d", t.fileSizeLimit) + if err := t.updateStatus(ctx, model.ConnectorStatusPending); err != nil { + return err + } + zap.S().Infof("send message to connector %s", t.connectorModel.Name) + return t.messenger.Publish(ctx, t.messenger.StreamConfig().ConnectorStreamName, + t.messenger.StreamConfig().ConnectorStreamSubject, data) +} + +func (t *trigger) UpToDate(ctx context.Context) error { + // may be to be implemented in future + return nil +} + +func NewTrigger(messenger messaging.Client, + connectorRepo repository.ConnectorRepository, + docRepo repository.DocumentRepository, + connectorModel *model.Connector, + fileSizeLimit int, + oauthURL string) *trigger { + return &trigger{ + messenger: messenger, + connectorRepo: connectorRepo, + docRepo: docRepo, + connectorModel: connectorModel, + fileSizeLimit: fileSizeLimit, + oauthURL: oauthURL, + tracer: otel.Tracer(model.TracerConnector), + } +} + +// update status of connector in database +func (t *trigger) updateStatus(ctx context.Context, status string) error { + t.connectorModel.Status = status + t.connectorModel.LastUpdate = pg.NullTime{time.Now().UTC()} + return t.connectorRepo.Update(ctx, t.connectorModel) +} diff --git a/src/backend/promtail-config.yaml b/src/backend/promtail-config.yaml new file mode 100644 index 00000000..ed06e8cd --- /dev/null +++ b/src/backend/promtail-config.yaml @@ -0,0 +1,18 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: +- job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log diff --git a/src/backend/semantic/.DS_Store b/src/backend/semantic/.DS_Store new file mode 100644 index 00000000..7252810f Binary files /dev/null and b/src/backend/semantic/.DS_Store differ diff --git a/src/backend/semantic/Dockerfile b/src/backend/semantic/Dockerfile new file mode 100644 index 00000000..ea7e39eb --- /dev/null +++ b/src/backend/semantic/Dockerfile @@ -0,0 +1,108 @@ +# # ################################### +# # Distroless working +# # ################################### +# FROM python:3.11.7-slim AS build-env + +# WORKDIR /app +# ADD ./ ./ + +# # Install any needed packages specified in requirements.txt +# RUN pip install --upgrade pip && \ +# pip install --no-cache-dir -r requirements.txt \ +# && rm -rf /root/.cache + +# # Install dependencies and clean up +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# && rm -rf /var/lib/apt/lists/* + +# # Use a distroless Python image as a parent image +# FROM gcr.io/distroless/python3-debian12 + +# # Copy the application code and installed packages from the build environment +# COPY --from=build-env /app /app +# COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages + +# # Set the working directory in the container +# WORKDIR /app + +# # Set environment variable for Python path +# ENV PYTHONPATH=/usr/local/lib/python3.11/site-package + +# # Run the application +# CMD ["semantic_service.py"] + + + + + +# ################################### +# python slim working +# ################################### +# using Python 3.11.0 as the newr supported by PyTorch +FROM python:3.11.7-slim-bookworm + + +# Set the working directory inside the container +WORKDIR /app + +# Copy the requirements.txt first to leverage Docker cache +COPY requirements.txt . + +# Install dependencies and clean up +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + && rm -rf /var/lib/apt/lists/* + + +# RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +RUN pip install --no-cache-dir -r requirements.txt \ + && rm -rf /root/.cache + +# the container before this 3 lines is 107mb, after it's 982mb!!!! +RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +RUN pip install transformers==4.40.2 tqdm==4.66.4 numpy==1.26.4 scikit-learn==1.4.2 scipy==1.13.0 nltk==3.8.1 pillow==10.3.0 + +# Copy the rest of the embedder application code +COPY . . + +# Command to run your application +CMD ["python", "semantic_service.py"] + + + + + + + + + + + + + +# # Use a slim Python base image +# # using Python 3.11.0 as the newr supported by PyTorch +# FROM python:3.11.7-slim-bookworm + + +# # Set the working directory inside the container +# WORKDIR /app + +# # Copy the requirements.txt first to leverage Docker cache +# COPY requirements.txt . + +# # Install dependencies and clean up +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# build-essential \ +# && rm -rf /var/lib/apt/lists/* +# RUN pip install --upgrade pip +# RUN pip install --no-cache-dir torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu +# RUN pip install --no-cache-dir -r requirements.txt \ +# && rm -rf /root/.cache + +# # Copy the rest of the embedder application code +# COPY . . + +# # Command to run your application +# CMD ["python", "semantic.py"] diff --git a/src/backend/semantic/__init__.py b/src/backend/semantic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/semantic/commands.txt b/src/backend/semantic/commands.txt new file mode 100644 index 00000000..9824fc69 --- /dev/null +++ b/src/backend/semantic/commands.txt @@ -0,0 +1,90 @@ + + +# build a container +sudo docker build . -t cognix/embedder:latest +sudo docker build . -t cognix/semantic:latest + +# run container +docker run --name embedder cognix/embedder:latest +docker run --name semantic cognix/semantic:latest + +#log inside a container +docker exec -it CONTAINER_NAME sh + +df + +# interactive shell +docker run -it --entrypoint /bin/sh cognix/embedder:latest + +# cleans a lot of space +docker system prune + +# resources (cpu/ram) usage by container +docker stats +docker stats {conteiner name} + +from cognix-services +sudo docker-compose -f deployment/docker-compose-cognix.yaml down +sudo docker-compose -f deployment/docker-compose-cognix.yaml up -d + +sudo docker-compose -f deployment/docker-compose-test.yaml up --build semantic --force-recreate + + +To generate python objects out of proto files +Attention this is the right command I got severe troubles +python -m grpc_tools.protoc -I . --python_out=. --pyi_out=. --grpc_python_out=. protos/embed_service.proto + +python3 -m grpc_tools.protoc -I . --python_out=. --pyi_out=. --grpc_python_out=. semantic_data.proto + + + + +You can solve the problem by set up "protoc" options. + +Go into settings > Extensions > vscode-proto3 configuration and then click Edit in settings.json. (you can just edit .vscode/settings.json too.) + +"protoc": { +"path": "/usr/local/bin/protoc", +"options": [ +"--proto_path=${workspaceRoot}/common/proto", +] +} + +python3 -m semantic.semantic_service + + +pod configuration + +apiVersion: v1 +kind: Pod +metadata: + name: sentence-transformer +spec: + containers: + - name: sentence-transformer + image: your-docker-image + resources: + requests: + memory: "4Gi" # Start with 4GiB for a medium-sized model + cpu: "2000m" # 2 vCPUs + limits: + memory: "8Gi" # Allow up to 8GiB for peak usage + cpu: "4000m" # Allow up to 4 vCPUs + +pycharm debug +Run to next breakpoint: F9 +Step Over: F8 +Step Into: F7 + + +view gpu perf +https://tlkh.github.io/asitop/ + +pip install asitop + +sudo asitop + + +docker run -d -p 3009:8080 -v ollama:/root/.ollama -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:ollama + + diff --git a/src/backend/semantic/lib/.DS_Store b/src/backend/semantic/lib/.DS_Store new file mode 100644 index 00000000..7a870099 Binary files /dev/null and b/src/backend/semantic/lib/.DS_Store differ diff --git a/src/backend/semantic/lib/__init__.py b/src/backend/semantic/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/semantic/lib/db/__init__.py b/src/backend/semantic/lib/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/semantic/lib/db/db_connector.py b/src/backend/semantic/lib/db/db_connector.py new file mode 100644 index 00000000..a821224d --- /dev/null +++ b/src/backend/semantic/lib/db/db_connector.py @@ -0,0 +1,133 @@ +from sqlalchemy import Column, BigInteger, UUID, TIMESTAMP, JSON, Enum, func, String +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from sqlalchemy.exc import OperationalError +from contextlib import contextmanager +import enum +import time +from lib.db.dc_connection_manager import ConnectionManager +from typing import List + +Base = declarative_base() + + +class Status(enum.Enum): + READY_TO_PROCESS = "READY_TO_PROCESS" + PENDING = "PENDING" + PROCESSING = "PROCESSING" + COMPLETED_SUCCESSFULLY = "COMPLETED_SUCCESSFULLY" + COMPLETED_WITH_ERRORS = "COMPLETED_WITH_ERRORS" + DISABLED = "DISABLED" + UNABLE_TO_PROCESS = "UNABLE_TO_PROCESS" + + +class Connector(Base): + __tablename__ = 'connectors' + + id = Column(BigInteger, primary_key=True, default=func.unique_rowid()) + name = Column(String, nullable=False) + type = Column(String(50), nullable=False) + connector_specific_config = Column(JSON, nullable=False) + refresh_freq = Column(BigInteger, nullable=True) + user_id = Column(UUID(as_uuid=True), nullable=False) + tenant_id = Column(UUID(as_uuid=True), nullable=True) + last_successful_analyzed = Column(TIMESTAMP(timezone=False), nullable=True) + status = Column(Enum(Status), nullable=True) + total_docs_analyzed = Column(BigInteger, nullable=False) + creation_date = Column(TIMESTAMP(timezone=False), nullable=False) + last_update = Column(TIMESTAMP(timezone=False), nullable=True) + deleted_date = Column(TIMESTAMP(timezone=False), nullable=True) + + def __repr__(self): + return (f"") + + +def with_retry(func): + def wrapper(*args, **kwargs): + retries = 3 + for i in range(retries): + try: + return func(*args, **kwargs) + except OperationalError as e: + if i < retries - 1: + time.sleep(2 ** i) # Exponential backoff + else: + raise e + + return wrapper + + +class ConnectorCRUD: + def __init__(self, connection_string: str): + self.connection_manager = ConnectionManager(connection_string) + + @contextmanager + def session_scope(self): + with self.connection_manager.get_session() as session: + yield session + + @with_retry + def insert_connector(self, **kwargs) -> int: + with self.session_scope() as session: + new_connector = Connector(**kwargs) + session.add(new_connector) + session.commit() + return new_connector.id + + @with_retry + def insert_connector_object(self, connector: Connector) -> int: + with self.session_scope() as session: + session.add(connector) + session.commit() + return connector.id + + @with_retry + def insert_connectors_batch(self, connectors: List[Connector]) -> List[Connector]: + with self.session_scope() as session: + session.add_all(connectors) + session.commit() + for connector in connectors: + session.refresh(connector) # Refresh each connector to get the IDs from the database + return connectors + + @with_retry + def select_connector(self, connector_id: int) -> Connector | None: + if connector_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + connector = session.query(Connector).filter_by(id=connector_id).first() + if connector: + session.expunge(connector) # Detach the instance from the session + return connector + + @with_retry + def update_connector(self, connector_id: int, **kwargs) -> int: + if connector_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + updated_connectors = session.query(Connector).filter_by(id=connector_id).update(kwargs) + session.commit() + return updated_connectors + + @with_retry + def delete_by_connector_id(self, connector_id: int) -> int: + if connector_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + deleted_connectors = session.query(Connector).filter_by(id=connector_id).delete() + session.commit() + return deleted_connectors + + @with_retry + def delete_by_tenant_id(self, tenant_id: UUID) -> int: + if not tenant_id: + raise ValueError("Tenant ID must be provided") + with self.session_scope() as session: + deleted_connectors = session.query(Connector).filter_by(tenant_id=tenant_id).delete() + session.commit() + return deleted_connectors diff --git a/src/backend/semantic/lib/db/db_document.py b/src/backend/semantic/lib/db/db_document.py new file mode 100644 index 00000000..67c2269a --- /dev/null +++ b/src/backend/semantic/lib/db/db_document.py @@ -0,0 +1,180 @@ +import os +import logging +from dotenv import load_dotenv +from sqlalchemy import Column, BigInteger, TIMESTAMP, Boolean, func, Text, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.dialects.postgresql import UUID +from lib.db.dc_connection_manager import ConnectionManager +from contextlib import contextmanager +from typing import List +from sqlalchemy.exc import OperationalError +import time + +load_dotenv() + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) + +logger = logging.getLogger(__name__) + +Base = declarative_base() + + +class Document(Base): + __tablename__ = 'documents' + + id = Column(Integer, primary_key=True, autoincrement=True) + parent_id = Column(BigInteger, nullable=True) + connector_id = Column(BigInteger, nullable=False) + source_id = Column(Text, nullable=False) + url = Column(Text, nullable=True) + signature = Column(Text, nullable=True) + chunking_session = Column(UUID(as_uuid=True), nullable=True) + analyzed = Column(Boolean, nullable=False, default=False) + creation_date = Column(TIMESTAMP(timezone=False), nullable=False, default=func.now()) + last_update = Column(TIMESTAMP(timezone=False), nullable=True) + + def __repr__(self): + return (f"") + + +def with_retry(func): + def wrapper(*args, **kwargs): + retries = 3 + for i in range(retries): + try: + return func(*args, **kwargs) + except OperationalError as e: + if i < retries - 1: + logger.warning(f"😱 cockroach falls in retry mode{e}") + time.sleep(2 ** i) # Exponential backoff + else: + raise e + + return wrapper + + +class DocumentCRUD: + def __init__(self, connection_string: str): + self.connection_manager = ConnectionManager(connection_string) + + @contextmanager + def session_scope(self): + with self.connection_manager.get_session() as session: + yield session + + @with_retry + def insert_document(self, **kwargs) -> int: + with self.session_scope() as session: + new_document = Document(**kwargs) + session.add(new_document) + session.commit() + return new_document.id + + @with_retry + def insert_document_object(self, document: Document) -> int: + with self.session_scope() as session: + session.add(document) + session.commit() + return document.id + + @with_retry + def insert_documents_list(self, documents: List[Document]) -> None: + """ + Inserts a list of Document objects into the database. + :param documents: List of Document objects. + """ + with self.session_scope() as session: + session.add_all(documents) + session.commit() # Commit the transaction to save changes # Commit the transaction to save changes + # for doc in new_documents: + # session.refresh(doc) # Refresh each document to get the IDs from the database + # return [doc.id for doc in new_documents] + + # @with_retry + # def insert_documents_batch(self, documents: List[Document]) -> List[Document]: + # with self.session_scope() as session: + # session.add_all(documents) + # session.commit() + # for document in documents: + # session.refresh(document) # Refresh each document to get the IDs from the database + # return documents + + @with_retry + def insert_documents_batch(self, documents: List[Document]) -> List[dict]: + with self.session_scope() as session: + session.add_all(documents) + session.commit() + document_data = [] + for document in documents: + session.refresh(document) # Refresh each document to get the IDs from the database + document_data.append({ + 'id': document.id, + 'url': document.url, + 'connector_id': document.connector_id, + 'chunking_session': document.chunking_session, + 'analyzed': document.analyzed, + 'creation_date': document.creation_date, + 'last_update': document.last_update + }) + return document_data + + @with_retry + def select_document(self, document_id: int) -> Document | None: + if document_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + document = session.query(Document).filter_by(id=document_id).first() + if document: + session.expunge(document) # Detach the instance from the session + return document + + @with_retry + def update_document(self, document_id: int, **kwargs) -> int: + if document_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + updated_docs = session.query(Document).filter_by(id=document_id).update(kwargs) + session.commit() + return updated_docs + + def update_document_object(self, document: Document): + if document.id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + existing_document = session.query(Document).filter_by(id=document.id).first() + if not existing_document: + raise ValueError("Document not found") + + # really afraid of these things!!! + existing_document.chunking_session = document.chunking_session + existing_document.analyzed = document.analyzed + existing_document.last_update = document.last_update + + session.commit() + + @with_retry + def delete_by_document_id(self, document_id: int) -> int: + if document_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + deleted_docs = session.query(Document).filter_by(id=document_id).delete() + session.commit() + return deleted_docs + + @with_retry + def delete_by_parent_id(self, parent_id: int) -> int: + if parent_id <= 0: + raise ValueError("ID value must be positive") + with self.session_scope() as session: + deleted_docs = session.query(Document).filter_by(parent_id=parent_id).delete() + session.commit() + return deleted_docs diff --git a/src/backend/semantic/lib/db/dc_connection_manager.py b/src/backend/semantic/lib/db/dc_connection_manager.py new file mode 100644 index 00000000..9fa697e7 --- /dev/null +++ b/src/backend/semantic/lib/db/dc_connection_manager.py @@ -0,0 +1,31 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, scoped_session +from contextlib import contextmanager + + +class ConnectionManager: + _instance = None + + def __new__(cls, connection_string=None): + if cls._instance is None: + cls._instance = super(ConnectionManager, cls).__new__(cls) + cls._instance._engine = create_engine( + connection_string, + pool_size=20, + max_overflow=0 + ) + # isolation_level="READ COMMITTED" # Set the isolation level to READ COMMITTED + cls._instance._session_factory = scoped_session(sessionmaker(bind=cls._instance._engine)) + return cls._instance + + @contextmanager + def get_session(self): + session = self._session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/src/backend/semantic/lib/db/jetstream_event_subscriber.py b/src/backend/semantic/lib/db/jetstream_event_subscriber.py new file mode 100644 index 00000000..d5e33074 --- /dev/null +++ b/src/backend/semantic/lib/db/jetstream_event_subscriber.py @@ -0,0 +1,143 @@ +import logging + +from google.protobuf import message as _message +from nats.aio.client import Client as NATS +from nats.aio.msg import Msg +from nats.js.api import ConsumerConfig, StreamConfig, AckPolicy, DeliverPolicy, RetentionPolicy +from nats.js.errors import BadRequestError + +from readiness_probe import ReadinessProbe + + +class JetStreamEventSubscriber: + def __init__(self, nats_url: str, stream_name: str, subject: str, + connect_timeout: int, reconnect_time_wait: int, + max_reconnect_attempts: int, ack_wait: int, + max_deliver: int, proto_message_type: _message.Message): + self.nats_url = nats_url + self.stream_name = stream_name + self.subject = subject + self.connect_timeout = connect_timeout + self.reconnect_time_wait = reconnect_time_wait + self.ack_wait = ack_wait + self.max_reconnect_attempts = max_reconnect_attempts + self.max_deliver = max_deliver + self.proto_message_type = proto_message_type + self.event_handler = None + self.nc = NATS() + self.js = None # needs to be created in connect_and_subscribe + self.logger = logging.getLogger(self.__class__.__name__) + + async def connect_and_subscribe(self): + try: + # Connect to NATS + # TODO: implement callbacks from connect <-------------------- DO THIS :) + self.logger.info(f"🔌 connecting to nats endpoint {self.nats_url} ..") + + await self.nc.connect(servers=[self.nats_url], + connect_timeout=self.connect_timeout, + reconnect_time_wait=self.reconnect_time_wait, + max_reconnect_attempts=self.max_reconnect_attempts) + # ping_interval=460000) + # https://github.com/nats-io/nats.py/discussions/299 + + self.logger.info(f"successfully connected {self.nats_url}") + + # Create JetStream context + self.js = self.nc.jetstream() + + # Create the stream configuration + stream_config = StreamConfig( + name=self.stream_name, + subjects=[self.subject], + # A work-queue retention policy satisfies a very common use case of queuing up messages that are + # intended to be processed once and only once. + # https://natsbyexample.com/examples/jetstream/workqueue-stream/go + retention=RetentionPolicy.WORK_QUEUE + # retention=RetentionPolicy.LIMITS + ) + + try: + await self.js.add_stream(stream_config) + except BadRequestError as e: + if e.code == 400: + self.logger.warning("😱 jetstream stream was using a different configuration. Destroying and " + "recreating with the right configuration") + try: + await self.js.delete_stream(stream_config.name) + await self.js.add_stream(stream_config) + self.logger.info("jetstream stream re-created successfully") + except Exception as e: + self.logger.exception(f"❌ Exception while deleting and recreating Jetstream: {e}") + except Exception as e: + self.logger.exception(f"❌ {e}") + raise e + + # Define consumer configuration + consumer_config = ConsumerConfig( + # durable_name="durable_chunkdata", do not set herem, it is set in pull_subscribe + ack_wait=self.ack_wait, # 3600 seconds + max_deliver=self.max_deliver, + ack_policy=AckPolicy.EXPLICIT, + # DeliverPolicy.ALL is mandatory when setting retention=RetentionPolicy.WORK_QUEUE for StreamConfig + deliver_policy=DeliverPolicy.ALL, + ) + + # Subscribe to the subject + try: + self.logger.info(f"subscribing to jetstream {self.stream_name} - {self.subject} ..") + psub = await self.js.pull_subscribe( + subject=self.subject, + stream=stream_config.name, + durable="worker", + config=consumer_config, + ) + self.logger.info(f"successfully subscribed to jetstream {self.stream_name} - {self.subject}") + + # psub.fetch() + while True: + try: + # await asyncio.sleep(2) + # notifying the readiness probe that the service is alive + ReadinessProbe().update_last_seen() + msgs = await psub.fetch(1, timeout=5) + self.logger.info(msgs) + for msg in msgs: + # ack will be done once the process is completed + # await msg.ack_sync() + await self.message_handler(msg) + self.logger.info(msg) + except TimeoutError: + self.logger.info("waiting for incoming events..") + pass + except Exception as e: + self.logger.error(f"❌ can't connect or subscribe to {self.nats_url} {self.stream_name} {self.subject} {e}") + raise e + + async def message_handler(self, msg: Msg): + try: + if self.event_handler: + await self.event_handler(msg) + except Exception as e: + self.logger.exception(f"❌ failed to process message: {e}") + + def set_event_handler(self, event_handler): + self.event_handler = event_handler + + async def close(self): + await self.nc.close() + + async def flush(self): + await self.nc.flush(2) + + async def disconnected_event(self): + self.logger.warning('😱 Got disconnected!') + + async def reconnected_event(self, nc: NATS) -> None: + self.logger.warning(f'😱 Got reconnected to {nc.connected_url.netloc}') + + async def error_event(self, e: Exception) -> None: + self.logger.error(f"❌there was an error: {e}") + + async def closed_event(self, nc: NATS) -> None: + self.logger.info("connection closed") diff --git a/src/backend/semantic/lib/db/jetstream_publisher.py b/src/backend/semantic/lib/db/jetstream_publisher.py new file mode 100644 index 00000000..ca12a5fa --- /dev/null +++ b/src/backend/semantic/lib/db/jetstream_publisher.py @@ -0,0 +1,87 @@ +import asyncio +from nats.aio.client import Client as NATS +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.gen_types.file_type_pb2 import FileType +from nats.errors import TimeoutError, NoRespondersError +from nats.js.api import StreamConfig, RetentionPolicy +from nats.js.errors import BadRequestError +import logging +import os +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +nats_url = os.getenv('NATS_CLIENT_URL', 'nats://127.0.0.1:4222') +nats_connect_timeout = int(os.getenv('NATS_CLIENT_CONNECT_TIMEOUT', '30')) +nats_reconnect_time_wait = int(os.getenv('NATS_CLIENT_RECONNECT_TIME_WAIT', '30')) +nats_max_reconnect_attempts = int(os.getenv('NATS_CLIENT_MAX_RECONNECT_ATTEMPTS', '3')) +semantic_stream_name = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_NAME', 'semantic') +semantic_stream_subject = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_SUBJECT', 'semantic_activity') +semantic_ack_wait = int(os.getenv('NATS_CLIENT_SEMANTIC_ACK_WAIT', '3600')) # seconds +semantic_max_deliver = int(os.getenv('NATS_CLIENT_SEMANTIC_MAX_DELIVER', '3')) + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + + +class JetStreamPublisher: + def __init__(self, subject, stream_name): + self.logger = None + self.subject = subject + self.stream_name = stream_name + self.nc = NATS() + self.js = None + logger.info(f"{semantic_stream_name} - {semantic_stream_subject}") + + async def connect(self): + # Connect to NATS + await self.nc.connect(servers=[nats_url], + connect_timeout=nats_connect_timeout, + reconnect_time_wait=nats_reconnect_time_wait, + max_reconnect_attempts=nats_max_reconnect_attempts) + # Create JetStream context + self.js = self.nc.jetstream() + + # Create the stream configuration + stream_config = StreamConfig( + name=self.stream_name, + subjects=[self.subject], + # A work-queue retention policy satisfies a very common use case of queuing up messages that are intended + # to be processed once and only once. https://natsbyexample.com/examples/jetstream/workqueue-stream/go + retention=RetentionPolicy.WORK_QUEUE + ) + + try: + await self.js.add_stream(stream_config) + except BadRequestError as e: + if e.code == 400: + self.logger.info( + "Jetstream stream was using a different configuration. Destroying and recreating with the right " + "configuration") + try: + await self.js.delete_stream(stream_config.name) + await self.js.add_stream(stream_config) + self.logger.info("Jetstream stream re-created successfully") + except Exception as e: + self.logger.exception(f"Exception while deleting and recreating Jetstream: {e}") + + async def publish(self, message): + try: + await self.js.publish(self.subject, message.SerializeToString()) + logger.info("Message published successfully!") + except NoRespondersError: + logger.error("❌ No responders available for request") + except TimeoutError: + logger.error("❌ Request to JetStream timed out") + except Exception as e: + logger.error(f"❌ Failed to publish message: {e}") + + async def close(self): + await self.nc.close() \ No newline at end of file diff --git a/src/backend/semantic/lib/db/milvus_db.py b/src/backend/semantic/lib/db/milvus_db.py new file mode 100644 index 00000000..32127610 --- /dev/null +++ b/src/backend/semantic/lib/db/milvus_db.py @@ -0,0 +1,192 @@ +import logging +import os +import time +from typing import List + +import grpc +from dotenv import load_dotenv +from numpy import int64 +from pymilvus import connections, utility, FieldSchema, CollectionSchema, DataType, Collection + +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.gen_types.embed_service_pb2 import EmbedRequest +from lib.gen_types.embed_service_pb2_grpc import EmbedServiceStub +from lib.spider.chunked_item import ChunkedItem + +# Load environment variables from .env file +load_dotenv() + +# Get nats url from env +milvus_alias = os.getenv("MILVUS_ALIAS", 'default') +milvus_host = os.getenv("MILVUS_HOST", "127.0.0.1") +milvus_port = os.getenv("MILVUS_PORT", "19530") +milvus_index_type = os.getenv("MILVUS_INDEX_TYPE", "DISKANN") +milvus_metric_type = os.getenv("MILVUS_METRIC_TYPE", "COSINE") + +milvus_user = "root" +milvus_pass = "sq5/6<$Y4aD`2;Gba'E#" + +embedder_grpc_host = os.getenv("EMBEDDER_GRPC_HOST", "localhost") +embedder_grpc_port = os.getenv("EMBEDDER_GRPC_PORT", "50051") + + +class Milvus_DB: + def __init__(self): + self.logger = logging.getLogger(self.__class__.__name__) + self._connect() + + def _connect(self): + try: + connections.connect( + alias=milvus_alias, + host=milvus_host, + port=milvus_port, + user=milvus_user, + password=milvus_pass + ) + # self.logger.info("Connected to Milvus") + except Exception as e: + self.logger.error(f"❌ Failed to connect to Milvus: {e}") + + def ensure_connection(self): + if not utility.connections.has_connection(milvus_alias): + self.logger.info("Reconnecting to Milvus") + self._connect() + + def delete_by_document_id_and_parent_id(self, document_id: int64, collection_name: str): + start_time = time.time() # Record the start time + # self.logger.info(f"deleting all entities related to document {document_id}") + self.ensure_connection() + try: + if utility.has_collection(collection_name): + collection = Collection(collection_name) # Get an existing collection. + self.logger.debug(f"collection: {collection_name} has {collection.num_entities} entities") + + # Create expressions to find matching entities + expr = f"document_id == {document_id} or parent_id == {document_id}" + + # Retrieve the primary keys of matching entities + results = collection.query(expr, output_fields=["id"]) + ids_to_delete = [res["id"] for res in results] + + if ids_to_delete: + # Delete entities by their primary keys + delete_expr = f"id in [{', '.join(map(str, ids_to_delete))}]" + collection.delete(delete_expr) + collection.flush() + self.logger.debug(f"deleted documents with document_id or parent_id: {document_id}") + else: + self.logger.debug(f"No documents found with document_id or parent_id: {document_id}") + except Exception as e: + self.logger.error(f"❌ failed to delete documents with document_id and parent_id {document_id}: {e}") + finally: + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + # self.logger.info(f"⏰ total elapsed time: {elapsed_time:.2f} seconds") + + def query(self, query: str, data: SemanticData) -> Collection: + start_time = time.time() # Record the start time + self.ensure_connection() + try: + collection = Collection(name=data.collection_name) + collection.load() + + embedding = self.embedd(query, data.model_name) + + result = collection.search( + data=[embedding], # Embed search value + anns_field="vector", # Search across embeddings + param={"metric_type": f"{milvus_metric_type}", "params": {"ef": 64}}, + limit=10, # Limit to top_k results per search + output_fields=["content"] + ) + + if self.logger.level == logging.DEBUG: + answer = "" + self.logger.debug("enumerating vector database results") + for i, hits in enumerate(result): + for hit in hits: + self.logger.debug( + f"Nearest Neighbor Number {i}: {hit.entity.get('sentence')} ---- {hit.distance}\n") + answer += hit.entity.get('sentence') + self.logger.debug("end enumeration") + return collection + except Exception as e: + self.logger.error(f"❌ {e}") + finally: + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + self.logger.debug(f"⏰🤖 milvus query total elapsed time: {elapsed_time:.2f} seconds") + + def store_chunk_list(self, chunk_list: List[ChunkedItem], collection_name: str, model_name: str, model_dimension: int): + + entities = [] + + connections.connect( + alias=milvus_alias, + host=milvus_host, + port=milvus_port, + user=milvus_user, + password=milvus_pass + ) + + fields = [ + FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True), + FieldSchema(name="document_id", dtype=DataType.INT64), + FieldSchema(name="parent_id", dtype=DataType.INT64), + FieldSchema(name="content", dtype=DataType.JSON), + FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim= model_dimension), + ] + + schema = CollectionSchema(fields=fields, enable_dynamic_field=True) + collection = Collection(name=collection_name, schema=schema) + + index_params = { + "index_type": milvus_index_type, + "metric_type": milvus_metric_type, + } + + collection.create_index(field_name="vector", index_params=index_params) + collection.load() + + for item in chunk_list: + # Check if the content exceeds milvus limit + if len(item.content) > 65535: + truncated_content = item.content[:65535] + else: + truncated_content = item.content + embedding = self.embedd(truncated_content, model_name) + json_content = {"content": truncated_content} + entities.append({ + "document_id": item.document_id, + "parent_id": item.parent_id, + "content": json_content, + "vector": embedding + }) + + collection.insert(entities) + collection.flush() + success = True + self.logger.debug(f"Elements successfully inserted in collection") + + def embedd(self, content_to_embedd: str, model: str) -> List[float]: + start_time = time.time() # Record the start time + with grpc.insecure_channel(f"{embedder_grpc_host}:{embedder_grpc_port}", + options=[ + ('grpc.max_send_message_length', 100 * 1024 * 1024), # 100 MB + ('grpc.max_receive_message_length', 100 * 1024 * 1024) # 100 MB + ] + ) as channel: + stub = EmbedServiceStub(channel) + + self.logger.debug("Calling gRPC Service GetEmbed - Unary") + + embed_request = EmbedRequest(content=content_to_embedd, model=model) + embed_response = stub.GetEmbeding(embed_request) + + self.logger.debug("GetEmbedding gRPC call received correctly") + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + self.logger.debug(f"⏰🤖total elapsed time to create embedding: {elapsed_time:.2f} seconds") + + return list(embed_response.vector) diff --git a/src/backend/semantic/lib/gen_types/__init__.py b/src/backend/semantic/lib/gen_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/semantic/lib/gen_types/embed_service_pb2.py b/src/backend/semantic/lib/gen_types/embed_service_pb2.py new file mode 100644 index 00000000..f964e15a --- /dev/null +++ b/src/backend/semantic/lib/gen_types/embed_service_pb2.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: embed_service.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x65mbed_service.proto\x12\ncom.cognix\".\n\x0c\x45mbedRequest\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\"\x1f\n\rEmbedResponse\x12\x0e\n\x06vector\x18\x01 \x03(\x02\x32T\n\x0c\x45mbedService\x12\x44\n\x0bGetEmbeding\x12\x18.com.cognix.EmbedRequest\x1a\x19.com.cognix.EmbedResponse\"\x00\x42\x1aZ\x18\x62\x61\x63kend/core/proto;protob\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embed_service_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z\030backend/core/proto;proto' + _globals['_EMBEDREQUEST']._serialized_start=35 + _globals['_EMBEDREQUEST']._serialized_end=81 + _globals['_EMBEDRESPONSE']._serialized_start=83 + _globals['_EMBEDRESPONSE']._serialized_end=114 + _globals['_EMBEDSERVICE']._serialized_start=116 + _globals['_EMBEDSERVICE']._serialized_end=200 +# @@protoc_insertion_point(module_scope) diff --git a/src/backend/semantic/lib/gen_types/embed_service_pb2.pyi b/src/backend/semantic/lib/gen_types/embed_service_pb2.pyi new file mode 100644 index 00000000..d8baa1dc --- /dev/null +++ b/src/backend/semantic/lib/gen_types/embed_service_pb2.pyi @@ -0,0 +1,20 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class EmbedRequest(_message.Message): + __slots__ = ("content", "model") + CONTENT_FIELD_NUMBER: _ClassVar[int] + MODEL_FIELD_NUMBER: _ClassVar[int] + content: str + model: str + def __init__(self, content: _Optional[str] = ..., model: _Optional[str] = ...) -> None: ... + +class EmbedResponse(_message.Message): + __slots__ = ("vector",) + VECTOR_FIELD_NUMBER: _ClassVar[int] + vector: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, vector: _Optional[_Iterable[float]] = ...) -> None: ... diff --git a/src/backend/semantic/lib/gen_types/embed_service_pb2_grpc.py b/src/backend/semantic/lib/gen_types/embed_service_pb2_grpc.py new file mode 100644 index 00000000..c23c9380 --- /dev/null +++ b/src/backend/semantic/lib/gen_types/embed_service_pb2_grpc.py @@ -0,0 +1,106 @@ + +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! + + + + +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import lib.gen_types.embed_service_pb2 as embed__service__pb2 + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in embed_service_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) + + +class EmbedServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetEmbeding = channel.unary_unary( + '/com.cognix.EmbedService/GetEmbeding', + request_serializer=embed__service__pb2.EmbedRequest.SerializeToString, + response_deserializer=embed__service__pb2.EmbedResponse.FromString, + _registered_method=True) + + +class EmbedServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def GetEmbeding(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_EmbedServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetEmbeding': grpc.unary_unary_rpc_method_handler( + servicer.GetEmbeding, + request_deserializer=embed__service__pb2.EmbedRequest.FromString, + response_serializer=embed__service__pb2.EmbedResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.cognix.EmbedService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class EmbedService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def GetEmbeding(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/com.cognix.EmbedService/GetEmbeding', + embed__service__pb2.EmbedRequest.SerializeToString, + embed__service__pb2.EmbedResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/backend/semantic/lib/gen_types/file_type_pb2.py b/src/backend/semantic/lib/gen_types/file_type_pb2.py new file mode 100644 index 00000000..95ade7b4 --- /dev/null +++ b/src/backend/semantic/lib/gen_types/file_type_pb2.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: file_type.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x66ile_type.proto\x12\ncom.cognix*\xff\x01\n\x08\x46ileType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x07\n\x03PDF\x10\x01\x12\x07\n\x03XPS\x10\x02\x12\x07\n\x03TXT\x10\x03\x12\x08\n\x04\x44OCX\x10\x04\x12\x07\n\x03\x44OC\x10\x05\x12\x08\n\x04XLSX\x10\x06\x12\x07\n\x03XLS\x10\x07\x12\x08\n\x04PPTX\x10\x08\x12\x07\n\x03PPT\x10\t\x12\x08\n\x04HWPX\x10\n\x12\x08\n\x04\x45PUB\x10\x0b\x12\x08\n\x04MOBI\x10\x0c\x12\x07\n\x03\x46\x42\x32\x10\r\x12\x07\n\x03\x43\x42Z\x10\x0e\x12\x06\n\x02MD\x10\x0f\x12\x07\n\x03URL\x10\x10\x12\x06\n\x02YT\x10\x11\x12\x07\n\x03MP4\x10\x12\x12\x07\n\x03MP3\x10\x13\x12\x08\n\x04MPEG\x10\x14\x12\x08\n\x04MPGA\x10\x15\x12\x07\n\x03M4A\x10\x16\x12\x07\n\x03WAV\x10\x17\x12\x08\n\x04WEBM\x10\x18\x12\x07\n\x03MOV\x10\x19\x42\x1aZ\x18\x62\x61\x63kend/core/proto;protob\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'file_type_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z\030backend/core/proto;proto' + _globals['_FILETYPE']._serialized_start=32 + _globals['_FILETYPE']._serialized_end=287 +# @@protoc_insertion_point(module_scope) diff --git a/src/backend/semantic/lib/gen_types/file_type_pb2.pyi b/src/backend/semantic/lib/gen_types/file_type_pb2.pyi new file mode 100644 index 00000000..1c1c38b1 --- /dev/null +++ b/src/backend/semantic/lib/gen_types/file_type_pb2.pyi @@ -0,0 +1,60 @@ +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from typing import ClassVar as _ClassVar + +DESCRIPTOR: _descriptor.FileDescriptor + +class FileType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + UNKNOWN: _ClassVar[FileType] + PDF: _ClassVar[FileType] + XPS: _ClassVar[FileType] + TXT: _ClassVar[FileType] + DOCX: _ClassVar[FileType] + DOC: _ClassVar[FileType] + XLSX: _ClassVar[FileType] + XLS: _ClassVar[FileType] + PPTX: _ClassVar[FileType] + PPT: _ClassVar[FileType] + HWPX: _ClassVar[FileType] + EPUB: _ClassVar[FileType] + MOBI: _ClassVar[FileType] + FB2: _ClassVar[FileType] + CBZ: _ClassVar[FileType] + MD: _ClassVar[FileType] + URL: _ClassVar[FileType] + YT: _ClassVar[FileType] + MP4: _ClassVar[FileType] + MP3: _ClassVar[FileType] + MPEG: _ClassVar[FileType] + MPGA: _ClassVar[FileType] + M4A: _ClassVar[FileType] + WAV: _ClassVar[FileType] + WEBM: _ClassVar[FileType] + MOV: _ClassVar[FileType] +UNKNOWN: FileType +PDF: FileType +XPS: FileType +TXT: FileType +DOCX: FileType +DOC: FileType +XLSX: FileType +XLS: FileType +PPTX: FileType +PPT: FileType +HWPX: FileType +EPUB: FileType +MOBI: FileType +FB2: FileType +CBZ: FileType +MD: FileType +URL: FileType +YT: FileType +MP4: FileType +MP3: FileType +MPEG: FileType +MPGA: FileType +M4A: FileType +WAV: FileType +WEBM: FileType +MOV: FileType diff --git a/src/backend/semantic/lib/gen_types/file_type_pb2_grpc.py b/src/backend/semantic/lib/gen_types/file_type_pb2_grpc.py new file mode 100644 index 00000000..fbb7fe62 --- /dev/null +++ b/src/backend/semantic/lib/gen_types/file_type_pb2_grpc.py @@ -0,0 +1,29 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in file_type_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) diff --git a/src/backend/semantic/lib/gen_types/semantic_data_pb2.py b/src/backend/semantic/lib/gen_types/semantic_data_pb2.py new file mode 100644 index 00000000..cd33e8ed --- /dev/null +++ b/src/backend/semantic/lib/gen_types/semantic_data_pb2.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: semantic_data.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import lib.gen_types.file_type_pb2 as file__type__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13semantic_data.proto\x12\ncom.cognix\x1a\x0f\x66ile_type.proto\"\xfa\x01\n\x0cSemanticData\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rurl_recursive\x18\x02 \x01(\x08\x12\x10\n\x08site_map\x18\x03 \x01(\t\x12\x1a\n\x12search_for_sitemap\x18\x04 \x01(\x08\x12\x13\n\x0b\x64ocument_id\x18\x05 \x01(\x03\x12\x14\n\x0c\x63onnector_id\x18\x06 \x01(\x03\x12\'\n\tfile_type\x18\x07 \x01(\x0e\x32\x14.com.cognix.FileType\x12\x17\n\x0f\x63ollection_name\x18\x08 \x01(\t\x12\x12\n\nmodel_name\x18\t \x01(\t\x12\x17\n\x0fmodel_dimension\x18\n \x01(\x05\x42\x1aZ\x18\x62\x61\x63kend/core/proto;protob\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'semantic_data_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z\030backend/core/proto;proto' + _globals['_SEMANTICDATA']._serialized_start=53 + _globals['_SEMANTICDATA']._serialized_end=303 +# @@protoc_insertion_point(module_scope) diff --git a/src/backend/semantic/lib/gen_types/semantic_data_pb2.pyi b/src/backend/semantic/lib/gen_types/semantic_data_pb2.pyi new file mode 100644 index 00000000..94483eff --- /dev/null +++ b/src/backend/semantic/lib/gen_types/semantic_data_pb2.pyi @@ -0,0 +1,30 @@ +import file_type_pb2 as _file_type_pb2 +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class SemanticData(_message.Message): + __slots__ = ("url", "url_recursive", "site_map", "search_for_sitemap", "document_id", "connector_id", "file_type", "collection_name", "model_name", "model_dimension") + URL_FIELD_NUMBER: _ClassVar[int] + URL_RECURSIVE_FIELD_NUMBER: _ClassVar[int] + SITE_MAP_FIELD_NUMBER: _ClassVar[int] + SEARCH_FOR_SITEMAP_FIELD_NUMBER: _ClassVar[int] + DOCUMENT_ID_FIELD_NUMBER: _ClassVar[int] + CONNECTOR_ID_FIELD_NUMBER: _ClassVar[int] + FILE_TYPE_FIELD_NUMBER: _ClassVar[int] + COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int] + MODEL_NAME_FIELD_NUMBER: _ClassVar[int] + MODEL_DIMENSION_FIELD_NUMBER: _ClassVar[int] + url: str + url_recursive: bool + site_map: str + search_for_sitemap: bool + document_id: int + connector_id: int + file_type: _file_type_pb2.FileType + collection_name: str + model_name: str + model_dimension: int + def __init__(self, url: _Optional[str] = ..., url_recursive: bool = ..., site_map: _Optional[str] = ..., search_for_sitemap: bool = ..., document_id: _Optional[int] = ..., connector_id: _Optional[int] = ..., file_type: _Optional[_Union[_file_type_pb2.FileType, str]] = ..., collection_name: _Optional[str] = ..., model_name: _Optional[str] = ..., model_dimension: _Optional[int] = ...) -> None: ... diff --git a/src/backend/semantic/lib/gen_types/semantic_data_pb2_grpc.py b/src/backend/semantic/lib/gen_types/semantic_data_pb2_grpc.py new file mode 100644 index 00000000..7e9e2f32 --- /dev/null +++ b/src/backend/semantic/lib/gen_types/semantic_data_pb2_grpc.py @@ -0,0 +1,29 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in semantic_data_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) diff --git a/src/backend/semantic/lib/helpers/device_checker.py b/src/backend/semantic/lib/helpers/device_checker.py new file mode 100644 index 00000000..8910cd8d --- /dev/null +++ b/src/backend/semantic/lib/helpers/device_checker.py @@ -0,0 +1,111 @@ +import os + +import torch +import logging + +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +# Get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) + +# Get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + +class DeviceChecker: + logger = logger + + @staticmethod + def check_device(): + DeviceChecker.logger.info(f"🤖🤖🤖🤖🤖🤖 PyTorch version: {torch.__version__}") + + # Check PyTorch has access to CUDA (NVIDIA's GPU architecture) + cuda_available = torch.cuda.is_available() + + # Check PyTorch has access to MPS (Metal Performance Shader, Apple's GPU architecture) + mps_built = torch.backends.mps.is_built() + mps_available = torch.backends.mps.is_available() + + # Determine the device to use + if cuda_available: + device = "cuda" + elif mps_available: + device = "mps" + else: + device = "cpu" + + DeviceChecker.logger.info(f"🤖🤖🤖🤖🤖🤖 Using device: {device}") + + # Display additional information based on the device + # if device == "cuda": + # cuda_version = torch.version.cuda + # DeviceChecker.logger.info(f"Is CUDA available? {cuda_available}") + # DeviceChecker.logger.info(f"CUDA version: {cuda_version}") + # elif device == "mps": + # DeviceChecker.logger.info(f"Is MPS (Metal Performance Shader) built? {mps_built}") + # DeviceChecker.logger.info(f"Is MPS available? {mps_available}") + + # Create data and send it to the device + x = torch.rand(size=(3, 4)).to(device) + # DeviceChecker.logger.info(f"Tensor on {device}: {x}") + + +# # To test the device checker +# if __name__ == "__main__": +# DeviceChecker.check_device() + +# import torch +# import logging +# +# class DeviceChecker: +# logger = logging.getLogger("DeviceChecker") +# logger.setLevel(logging.INFO) +# handler = logging.StreamHandler() +# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# handler.setFormatter(formatter) +# logger.addHandler(handler) +# +# @staticmethod +# def check_device(): +# DeviceChecker.logger.info(f"PyTorch version: {torch.__version__}") +# +# # Check PyTorch has access to CUDA (NVIDIA's GPU architecture) +# cuda_available = torch.cuda.is_available() +# +# # Check PyTorch has access to MPS (Metal Performance Shader, Apple's GPU architecture) +# mps_built = torch.backends.mps.is_built() +# mps_available = torch.backends.mps.is_available() +# +# # Determine the device to use +# if cuda_available: +# device = "cuda" +# elif mps_available: +# device = "mps" +# else: +# device = "cpu" +# +# DeviceChecker.logger.info(f"🤖🤖🤖🤖🤖🤖 using device: {device}") +# +# # Display additional information based on the device +# # if device == "cuda": +# # cuda_version = torch.version.cuda +# # DeviceChecker.logger.info(f"Is CUDA available? {cuda_available}") +# # DeviceChecker.logger.info(f"CUDA version: {cuda_version}") +# # elif device == "mps": +# # DeviceChecker.logger.info(f"Is MPS (Metal Performance Shader) built? {mps_built}") +# # DeviceChecker.logger.info(f"Is MPS available? {mps_available}") +# +# # Create data and send it to the device +# x = torch.rand(size=(3, 4)).to(device) +# # DeviceChecker.logger.info(f"Tensor on {device}: {x}") +# +# # # To test the device checker +# # if __name__ == "__main__": +# # DeviceChecker.check_device() diff --git a/src/backend/semantic/lib/semantic/__init__.py b/src/backend/semantic/lib/semantic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/semantic/lib/semantic/markdown_extractor.py b/src/backend/semantic/lib/semantic/markdown_extractor.py new file mode 100644 index 00000000..cffa5517 --- /dev/null +++ b/src/backend/semantic/lib/semantic/markdown_extractor.py @@ -0,0 +1,178 @@ +import os +import sys +import time +import pymupdf4llm +import mistune +from typing import List, Dict, Any + + +class MarkdownSectionExtractor: + def __init__(self): + pass + + def _parse_markdown_content(self, markdown_content: str) -> List[Dict[str, Any]]: + markdown = mistune.create_markdown(renderer='ast') + return markdown(markdown_content) + + def extract_sections(self, markdown_content: str) -> List[Dict[str, Any]]: + parsed_content = self._parse_markdown_content(markdown_content) + sections = [] + current_section = [] + heading_hierarchy = [] + + def add_section(): + if current_section: + sections.append({ + 'headings': heading_hierarchy[:], # Copy of current heading hierarchy + 'content': current_section[:] + }) + current_section.clear() + + for element in parsed_content: + if element['type'] == 'heading': + level = element['attrs']['level'] + text = element['children'][0]['raw'] if element['children'] else '' + add_section() # Finish the current section before starting a new heading + + # Update heading hierarchy based on the level + if level == 1: + heading_hierarchy = [text] + else: + if len(heading_hierarchy) >= level: + heading_hierarchy = heading_hierarchy[:level - 1] + heading_hierarchy.append(text) + + current_section.append(element) + else: + current_section.append(element) + + add_section() # Add the last section + + return sections + + def extract_chunks(self, markdown: str) -> list[str]: + sections = self.extract_sections(markdown) + results = [] + # Process the extracted sections + for idx, section in enumerate(sections): + content_texts = [] + for element in section['content']: + if 'children' in element and element['children']: + for child in element['children']: + if 'raw' in child: + content_texts.append(child['raw']) + elif 'text' in child: + content_texts.append(child['text']) + elif 'raw' in element: + content_texts.append(element['raw']) + elif 'text' in element: + content_texts.append(element['text']) + + readable_text = ' '.join(content_texts) + headings_text = ' > '.join(section['headings']) + result = f"{headings_text}\n{readable_text}\n" + results.append(result) + return results + +import os +import sys +import time +import pymupdf4llm +import mistune +from typing import List, Dict, Any + + +class MarkdownSectionExtractor: + def __init__(self): + pass + + def _parse_markdown_content(self, markdown_content: str) -> List[Dict[str, Any]]: + markdown = mistune.create_markdown(renderer='ast') + return markdown(markdown_content) + + def extract_sections(self, markdown_content: str) -> List[Dict[str, Any]]: + parsed_content = self._parse_markdown_content(markdown_content) + sections = [] + current_section = [] + heading_hierarchy = [] + + def add_section(): + if current_section: + sections.append({ + 'headings': heading_hierarchy[:], # Copy of current heading hierarchy + 'content': current_section[:] + }) + current_section.clear() + + for element in parsed_content: + if element['type'] == 'heading': + level = element['attrs']['level'] + text = element['children'][0]['raw'] if element['children'] else '' + add_section() # Finish the current section before starting a new heading + + # Update heading hierarchy based on the level + if level == 1: + heading_hierarchy = [text] + else: + if len(heading_hierarchy) >= level: + heading_hierarchy = heading_hierarchy[:level - 1] + heading_hierarchy.append(text) + + current_section.append(element) + else: + current_section.append(element) + + add_section() # Add the last section + + return sections + + def extract_chunks(self, markdown: str) -> list[str]: + sections = self.extract_sections(markdown) + results = [] + # Process the extracted sections + for idx, section in enumerate(sections): + content_texts = [] + for element in section['content']: + if 'children' in element and element['children']: + for child in element['children']: + if 'raw' in child: + content_texts.append(child['raw']) + elif 'text' in child: + content_texts.append(child['text']) + elif 'raw' in element: + content_texts.append(element['raw']) + elif 'text' in element: + content_texts.append(element['text']) + + readable_text = ' '.join(content_texts) + headings_text = ' > '.join(section['headings']) + result = f"{headings_text}\n{readable_text}\n" + results.append(result) + return results + +# if __name__ == "__main__": +# import pathlib +# +# try: +# filename = sys.argv[1] +# except IndexError: +# print(f"Usage:\npython {os.path.basename(__file__)} input.pdf") +# sys.exit() +# +# t0 = time.perf_counter() +# +# markdown_content = pymupdf4llm.to_markdown(filename) +# print(markdown_content) +# +# extractor = MarkdownSectionExtractor() +# # sections = extractor.extract_sections(markdown_content) +# results = extractor.extract_chunks(markdown_content) +# +# +# # Print the results +# for result in results: +# print(result) +# print("****************** new section *********************\n") +# +# t1 = time.perf_counter() +# print(f"Markdown creation time for {filename=} {round(t1 - t0, 2)} sec.") diff --git a/src/backend/semantic/lib/semantic/semantic_base.py b/src/backend/semantic/lib/semantic/semantic_base.py new file mode 100644 index 00000000..c3c45881 --- /dev/null +++ b/src/backend/semantic/lib/semantic/semantic_base.py @@ -0,0 +1,186 @@ +import datetime +import os +import logging +import time +import uuid + +from minio import Minio + +from lib.db.db_document import Document, DocumentCRUD +from lib.db.milvus_db import Milvus_DB +from lib.gen_types.semantic_data_pb2 import SemanticData +from dotenv import load_dotenv + +from lib.spider.chunked_item import ChunkedItem +from lib.semantic.text_splitter import TextSplitter +from readiness_probe import ReadinessProbe + +# Load environment variables from .env file +load_dotenv() + +chunk_size = int(os.getenv('CHUNK_SIZE', 500)) +chunk_overlap = int(os.getenv('CHUNK_OVERLAP', 3)) +temp_path = os.getenv('LOCAL_TEMP_PATH', "../temp") + +minio_endpoint = os.getenv('MINIO_ENDPOINT', "minio:9000") +minio_access_key = os.getenv('MINIO_ACCESS_KEY', "minioadmin") +minio_secret_key = os.getenv('MINIO_SECRET_ACCESS_KEY', "minioadmin") +minio_use_ssl = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true' +semantic_stream_name = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_NAME', 'semantic') +semantic_stream_subject = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_SUBJECT', 'semantic_activity') + + +class BaseSemantic: + def __init__(self): + self.logger = logging.getLogger(self.__class__.__name__) + self.temp_path = temp_path + self.minio_endpoint = minio_endpoint + self.minio_access_key = minio_access_key + self.minio_secret_key = minio_secret_key + self.minio_use_ssl = minio_use_ssl + self.semantic_stream_name = semantic_stream_name + self.semantic_stream_subject = semantic_stream_subject + # Create an instance of TextSplitter + self.text_splitter = TextSplitter() + + async def analyze(self, data: SemanticData, full_process_start_time: float, ack_wait: int, + cockroach_url: str) -> int: + raise NotImplementedError("Chunk method needs to be implemented by subclasses") + + def keep_processing(self, full_process_start_time: float, ack_wait: int) -> bool: + # it returns true if the difference between start_time and now is less than ack_wait + # it returns false if the difference between start_time and now is equal or greater than ack_wait + end_time = time.time() # Record the end time + elapsed_time = end_time - full_process_start_time + return elapsed_time < ack_wait + + def store_collected_data(self, collected_data: list[ChunkedItem], data: SemanticData, document_crud: DocumentCRUD, + chunking_session: uuid, ack_wait: int, full_process_start_time: float): + start_time = time.time() # Record the start time + + # verifies if the method is taking longer than ack_wait + # if so we have to stop + if not self.keep_processing(full_process_start_time=full_process_start_time, ack_wait=ack_wait): + raise Exception(f"exceeded maximum processing time defined in NATS_CLIENT_SEMANTIC_ACK_WAIT of {ack_wait}") + + milvus_db = Milvus_DB() + # delete previous added chunks and vectors + # it deletes all the entries in Milvus related to the document which means it delete the document and + # any related child (by parent_id) + milvus_db.delete_by_document_id_and_parent_id(document_id=data.document_id, + collection_name=data.collection_name) + + # delete previous added child (chunks) documents + # we are not storing child docs anymore + # this means tha in cockroach we have only the reference to the original document + # and not about the single chunk + # document_crud.delete_by_parent_id(data.document_id) + + # updating the status of the doc + doc = document_crud.select_document(data.document_id) + doc.chunking_session = chunking_session + doc.analyzed = False + doc.last_update = datetime.datetime.utcnow() + document_crud.update_document_object(doc) + + logging.info(f"storing in milvus {len(collected_data)} entities for {data.url}") + + # notifying the readiness probe that the service is alive + ReadinessProbe().update_last_seen() + + # verifies if the method is taking longer than ack_wait + # if so we have to stop + if not self.keep_processing(full_process_start_time=full_process_start_time, ack_wait=ack_wait): + raise Exception( + f"exceeded maximum processing time defined in NATS_CLIENT_SEMANTIC_ACK_WAIT of {ack_wait}") + + # Perform batch insertion into Milvus + + milvus_db.store_chunk_list(chunk_list=collected_data, collection_name=data.collection_name, + model_name=data.model_name, model_dimension=data.model_dimension) + + # update the status of the doc in the relational db + doc.analyzed = True + doc.last_update = datetime.datetime.utcnow() + document_crud.update_document_object(doc) + + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + # self.logger.info(f"⏰🤖 total milvus and cockroach ops {elapsed_time}") + + def store_collected_data_none(self, data: SemanticData, document_crud: DocumentCRUD, chunking_session: uuid): + # storing in the db the item setting analyzed = false because we were not able to extract any text out of it + # there will be no trace of it in milvus + doc = Document(parent_id=data.document_id, connector_id=data.connector_id, source_id=data.url, + url=data.url, chunking_session=chunking_session, analyzed=False, + creation_date=datetime.datetime.utcnow(), last_update=datetime.datetime.utcnow()) + document_crud.update_document_object(doc) + + def log_end(self, collected_items, start_time): + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + self.logger.info(f"⏰ total elapsed time: {elapsed_time:.2f} seconds") + self.logger.info(f"📖 number of docs analyzed: {collected_items}") + + def delete_from_storages(self, url: str) -> None: + try: + """ + Delete a file from MinIO using the provided URL. + + :param url: The MinIO URL of the file to be deleted. + """ + # Extract bucket name and object name from the URL + parts = url.split(':') + bucket_name = parts[1] + object_name = parts[-1] + # Extract the file name from the object name + file_name = object_name.split('-')[-1] + # Combine the temporary path and the file name + local_path = os.path.join(self.temp_path, file_name) + + # Initialize the MinIO client + client = Minio( + self.minio_endpoint, + access_key=self.minio_access_key, + secret_key=self.minio_secret_key, + secure=self.minio_use_ssl # Use SSL if minio_use_ssl is true + ) + + # Delete the file from the bucket + client.remove_object(bucket_name, object_name) + self.logger.info(f"File {object_name} deleted successfully from bucket {bucket_name}") + os.remove(local_path) + self.logger.info(f"File {local_path} deleted successfully from temp storage") + except Exception as e: + error_message = str(e) if e else "Unknown error occurred" + self.logger.error(f"❌ {error_message}") + + def download_from_minio(self, url: str) -> str: + """ + Download a file from MinIO using the provided URL and save it to the specified local temporary path. + + :param url: The MinIO URL of the file to be downloaded. + :return: The full path to the downloaded file. + """ + # Extract bucket name and object name from the URL + parts = url.split(':') + bucket_name = parts[1] + object_name = parts[-1] + + # Extract the file name from the object name + file_name = object_name.split('-')[-1] + # Combine the temporary path and the file name + save_path = os.path.join(self.temp_path, file_name) + + # Initialize the MinIO client + client = Minio( + self.minio_endpoint, + access_key=self.minio_access_key, + secret_key=self.minio_secret_key, + secure=self.minio_use_ssl # Use SSL if minio_use_ssl is true + ) + + # Download the file from the bucket + client.fget_object(bucket_name, object_name, save_path) + print(f"File downloaded successfully and saved to {save_path}") + return save_path diff --git a/src/backend/semantic/lib/semantic/semantic_factory.py b/src/backend/semantic/lib/semantic/semantic_factory.py new file mode 100644 index 00000000..1a2d0350 --- /dev/null +++ b/src/backend/semantic/lib/semantic/semantic_factory.py @@ -0,0 +1,27 @@ +from lib.semantic.semantic_base import BaseSemantic +from lib.semantic.semantic_url import URLSemantic +from lib.semantic.semantic_generic import GenericSemantic +from lib.gen_types.file_type_pb2 import FileType +from lib.semantic.semantic_youtube import YTSemantic + + +class SemanticFactory: + # Define ranges for connector file types + generic_semantic_type_range = range(FileType.UNKNOWN, FileType.MD + 1) + + factories = { + FileType.URL: URLSemantic, + FileType.YT: YTSemantic, + # Additional mappings can be added here + } + + @staticmethod + def create_semantic_analyzer(file_type: FileType) -> BaseSemantic: + if file_type in SemanticFactory.generic_semantic_type_range: + return GenericSemantic() + + semantic_class = SemanticFactory.factories.get(file_type) + if not semantic_class: + raise ValueError(f"Unsupported file type: {file_type}") + + return semantic_class() diff --git a/src/backend/semantic/lib/semantic/semantic_generic.py b/src/backend/semantic/lib/semantic/semantic_generic.py new file mode 100644 index 00000000..25143e1e --- /dev/null +++ b/src/backend/semantic/lib/semantic/semantic_generic.py @@ -0,0 +1,87 @@ +import os +import time +import uuid + +import pymupdf4llm + +from lib.db.db_document import DocumentCRUD +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.semantic.markdown_extractor import MarkdownSectionExtractor +from lib.semantic.semantic_base import BaseSemantic +from lib.semantic.text_splitter import TextSplitter +from lib.spider.chunked_item import ChunkedItem +from minio import Minio +from minio.error import S3Error + + +class GenericSemantic(BaseSemantic): + async def analyze(self, data: SemanticData, full_process_start_time: float, ack_wait: int, + cockroach_url: str) -> int: + collected_items = 0 + # TODO: move all the time.time to perf_counter() + t0 = time.perf_counter() + + try: + # downloads the file from minio and stores locally + downloaded_file_path = self.download_from_minio(data.url) + file_type = "" + # Log the file type and size + if os.path.exists(downloaded_file_path): + file_type = os.path.splitext(downloaded_file_path)[1] + file_size = os.path.getsize(downloaded_file_path) + self.logger.info(f"analyzing a: {file_type} file, size: {file_size / 1024:.2f} KB") + else: + raise FileNotFoundError(f"File {downloaded_file_path} does not exist.") + + # Check if the file is a Markdown file + if file_type == '.md': + with open(downloaded_file_path, 'r') as file: + markdown_content = file.read() + else: + # Converts the file to Markdown using pymupdf + markdown_content = pymupdf4llm.to_markdown(downloaded_file_path) + + + # # detracts markdown sections with headers ready to be stored in chunks + # # on the vector and relational db + # extractor = MarkdownSectionExtractor() + # results = extractor.extract_chunks(markdown_content) + # + # # converting results to alist of ChunkedItems to that it can be passed + # # to the store and collect method + # collected_data = ChunkedItem.create_chunked_items(results=results, url=data.url, + # document_id=data.document_id, parent_id=0) + + collected_data = TextSplitter.create_chunked_items(content=markdown_content, url=data.url, document_id=data.document_id, parent_id=0) + # Failed to analyze_doc data: Failed to open file '../temp/file_example_XLS_5000.xls'. + # Failed to analyze_doc data: Failed to open file '../temp/file_example_XLS_10.XLS + # Failed to analyze_doc data: Failed to open file '../temp/file_example_PPT_1MB.ppt + # Failed to analyze_doc data: Failed to open file '../temp/file_example_PPT_250kB.ppt.ppt'. + # int: + collected_items = 0 + try: + spider = BS4Spider(data.url) + chunking_session = uuid.uuid4() + document_crud = DocumentCRUD(cockroach_url) + + # region scan for links and send messages + if data.url_recursive: + links = spider.extract_links(data.url) + documents_to_insert: List[Document] = [] + + if not links: + self.logger.warning( + f"😱 BS4Spider was not able to retrieve any content for {data.url}, switching to " + f"SeleniumSpider") + self.logger.warning( + "😱 SeleniumSpider is disabled, shall be re-enabled and tested as it is not working 100%") + + for link in links: + documents_to_insert.append( + Document(parent_id=data.document_id, connector_id=data.connector_id, source_id="", + url=link, + signature="", + chunking_session=chunking_session, analyzed=False)) + + self.logger.info("insert_documents_batch") + documents_to_send = document_crud.insert_documents_batch(documents=documents_to_insert) + + self.logger.info("JetStreamPublisher") + publisher = JetStreamPublisher(subject=self.semantic_stream_subject, + stream_name=self.semantic_stream_name) + self.logger.info("connect") + await publisher.connect() + + self.logger.info("for") + # for doc in documents_to_send: + # self.logger.info(f"doc id {doc.id}") + # semantic_data = SemanticData( + # url=doc.url, + # document_id=doc.id, + # url_recursive=False, + # connector_id=data.connector_id, + # file_type=FileType.URL, + # collection_name=data.collection_name) + for doc in documents_to_send: + self.logger.info(f"doc id {doc['id']}") + semantic_data = SemanticData( + url=doc['url'], + document_id=doc['id'], + url_recursive=False, + connector_id=doc['connector_id'], + file_type=FileType.URL, + collection_name=data.collection_name, + model_name=data.model_name, + model_dimension=data.model_dimension) + + await publisher.publish(semantic_data) + + self.logger.info("✉️ sending message to jetstream") + + await publisher.close() + + collected_items = 1 + # endregion + # region analyze single url + else: + self.logger.info(f"extracting content from: {data.url}") + + content = spider.process_page(url=data.url) + + chunking_session = uuid.uuid4() + document_crud = DocumentCRUD(cockroach_url) + + if not content: + self.logger.warning(f"😱no content found in {data.url}") + + if content: + collected_data = TextSplitter.create_chunked_items(content=content, url=data.url, + document_id=data.document_id, parent_id=0) + + collected_items = len(collected_data) + self.store_collected_data(data=data, document_crud=document_crud, + collected_data=collected_data, + chunking_session=chunking_session, + ack_wait=ack_wait, + full_process_start_time=full_process_start_time) + else: + self.store_collected_data_none(data=data, document_crud=document_crud, + chunking_session=chunking_session) + # endregion + except Exception as e: + self.logger.error(f"❌ Failed to process semantic data: {e}") + finally: + return collected_items + + diff --git a/src/backend/semantic/lib/semantic/semantic_youtube.py b/src/backend/semantic/lib/semantic/semantic_youtube.py new file mode 100644 index 00000000..801a8299 --- /dev/null +++ b/src/backend/semantic/lib/semantic/semantic_youtube.py @@ -0,0 +1,97 @@ +import time +import uuid +from typing import List, Dict, Optional +from urllib.parse import urlparse, parse_qs + +from youtube_transcript_api import YouTubeTranscriptApi + +from lib.db.db_document import DocumentCRUD +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.semantic.semantic_base import BaseSemantic +from lib.semantic.text_splitter import TextSplitter +from lib.spider.chunked_item import ChunkedItem + + +class YTSemantic(BaseSemantic): + def get_video_id(self, youtube_url: str) -> Optional[str]: + """ + Extracts the video ID from a YouTube URL. + + Args: + youtube_url (str): The URL of the YouTube video. + + Returns: + Optional[str]: The video ID if found, else None. + """ + parsed_url = urlparse(youtube_url) + if parsed_url.hostname == 'youtu.be': + return parsed_url.path[1:] + if parsed_url.hostname in ['www.youtube.com', 'youtube.com']: + if parsed_url.path == '/watch': + return parse_qs(parsed_url.query).get('v', [None])[0] + if parsed_url.path[:7] == '/embed/': + return parsed_url.path.split('/')[2] + if parsed_url.path[:3] == '/v/': + return parsed_url.path.split('/')[2] + return None + + def get_youtube_transcript(self, video_url: str) -> str | None: #Optional[List[Dict[str, str]]]: + """ + Fetches the transcript of a YouTube video. + + Args: + video_url (str): The URL of the YouTube video. + + Returns: + Optional[List[Dict[str, str]]]: The list of transcript entries if successful, else None. + """ + video_id = self.get_video_id(video_url) + if not video_id: + print("Invalid YouTube URL") + return None + try: + transcript_list = YouTubeTranscriptApi.get_transcript(video_id) + # return transcript_list + transcript = "" + for segment in transcript_list: + transcript += segment['text'] + "\n" + return transcript.strip() + except Exception as e: + self.logger.error(f"❌ {e}") + return None + + async def analyze(self, data: SemanticData, full_process_start_time: float, ack_wait: int, cockroach_url: str) -> int: + collected_items = 0 + try: + start_time = time.time() # Record the start time + self.logger.info(f"extracting transcript from: {data.url}") + + content = self.get_youtube_transcript(data.url) + + chunking_session = uuid.uuid4() + document_crud = DocumentCRUD(cockroach_url) + + if not content: + self.logger.warning(f"😱no content found in {data.url}") + + if content: + # TODO: VERY IMPORTANT + # We need content's summarization + + collected_data = TextSplitter.create_chunked_items(content=content, url=data.url, + document_id=data.document_id, parent_id=0) + + collected_items = len(collected_data) + self.store_collected_data(data=data, document_crud=document_crud, + collected_data=collected_data, + chunking_session=chunking_session, + ack_wait=ack_wait, + full_process_start_time=full_process_start_time) + # self.logger.debug(f"transcript \n {content}") + else: + self.store_collected_data_none(data=data, document_crud=document_crud, + chunking_session=chunking_session) + except Exception as e: + self.logger.error(f"❌ Failed to process semantic data: {e}") + finally: + return collected_items diff --git a/src/backend/semantic/lib/semantic/text_splitter.py b/src/backend/semantic/lib/semantic/text_splitter.py new file mode 100644 index 00000000..ae8a2d47 --- /dev/null +++ b/src/backend/semantic/lib/semantic/text_splitter.py @@ -0,0 +1,34 @@ +import os +from langchain_text_splitters import RecursiveCharacterTextSplitter +import logging +from typing import List, Tuple + +from lib.spider.chunked_item import ChunkedItem + +class TextSplitter: + chunk_size = int(os.getenv('CHUNK_SIZE', 500)) + chunk_overlap = int(os.getenv('CHUNK_OVERLAP', 3)) + + @classmethod + def create_chunked_items(cls, content: str, url: str, document_id: int, parent_id: int) -> List['ChunkedItem']: + chunked_items = [] + custom_text_splitter = RecursiveCharacterTextSplitter( + chunk_size=cls.chunk_size, + chunk_overlap=cls.chunk_overlap, + length_function=len, + separators=['\n'] + ) + + texts = custom_text_splitter.create_documents([content]) + + if texts: + logging.info(f"created {len(texts)} chunks for {url}") + else: + logging.info(f"no chunk created for {url}") + + for chunk in texts: + if chunk: + chunked_items.append(ChunkedItem(content=chunk.page_content, url=url, document_id=document_id, parent_id=parent_id)) + + return chunked_items + diff --git a/src/backend/semantic/lib/spider/__init__.py b/src/backend/semantic/lib/spider/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/semantic/lib/spider/chunked_item.py b/src/backend/semantic/lib/spider/chunked_item.py new file mode 100644 index 00000000..546fd07b --- /dev/null +++ b/src/backend/semantic/lib/spider/chunked_item.py @@ -0,0 +1,16 @@ +from typing import List + + +class ChunkedItem: + def __init__(self, content: str, url: str, document_id: int = None, parent_id: int = None): + self.url = url + self.content = content + self.document_id = document_id + self.parent_id = parent_id + + @classmethod + def create_chunked_items(cls, content: List[str], url: str, document_id: int, parent_id: int) -> List['ChunkedItem']: + return [cls(url, result, document_id, parent_id) for result in content] + + def __repr__(self): + return f"ChunkedItem(url={self.url}, content={self.content}, document_id={self.document_id}, parent_id={self.parent_id})" diff --git a/src/backend/semantic/lib/spider/spider_bs4.py b/src/backend/semantic/lib/spider/spider_bs4.py new file mode 100644 index 00000000..05e3a3fe --- /dev/null +++ b/src/backend/semantic/lib/spider/spider_bs4.py @@ -0,0 +1,102 @@ +import time +from typing import List, Optional +import requests +from bs4 import BeautifulSoup, Tag +from urllib.parse import urljoin, urlparse +import logging +from readiness_probe import ReadinessProbe + + +class BS4Spider: + def __init__(self, base_url: str) -> None: + self.visited: set[str] = set() + self.collected_data: List[str] = [] # Change type to List[str] + self.base_domain: str = urlparse(base_url).netloc + self.logger = logging.getLogger(__name__) + + def process_page(self, url: str) -> str: # Change return type to str + start_time: float = time.time() + + # Notifying the readiness probe that the service is alive + ReadinessProbe().update_last_seen() + + # Fetch and parse the URL + soup = self.fetch_and_parse(url) + if not soup: + return "" + + # Extract data from the page + page_content = self.extract_data(soup) + + end_time: float = time.time() # Record the end time + elapsed_time: float = end_time - start_time + self.logger.info(f"⏰ total elapsed time: {elapsed_time:.2f} seconds") + + return page_content # Return as a single string + + def extract_links(self, start_url: str) -> List[str]: + start_time: float = time.time() + links_to_visit: List[str] = [start_url] + all_links: List[str] = [] + + while links_to_visit: + url = links_to_visit.pop() + if url in self.visited: + continue + + # Notifying the readiness probe that the service is alive + ReadinessProbe().update_last_seen() + + # Fetch and parse the URL + soup: Optional[BeautifulSoup] = self.fetch_and_parse(url) + if not soup: + continue + + self.visited.add(url) + all_links.append(url) + + # Extract all links from the page + links: List[str] = [a['href'] for a in soup.find_all('a', href=True)] + for link in links: + # Convert relative links to absolute links + absolute_link: str = urljoin(url, link) + parsed_link = urlparse(absolute_link) + # Check if the link is an HTTP/HTTPS link, not visited yet, and does not contain a fragment + if parsed_link.scheme in ['http', + 'https'] and absolute_link not in self.visited and not parsed_link.fragment: + # Ensure the link is within the same domain + if parsed_link.netloc == self.base_domain: + links_to_visit.append(absolute_link) + + end_time: float = time.time() # Record the end time + elapsed_time: float = end_time - start_time + self.logger.info(f"⏰ total elapsed time: {elapsed_time:.2f} seconds") + + return all_links + + def fetch_and_parse(self, url: str) -> Optional[BeautifulSoup]: + try: + self.logger.info(f"Processing URL: {url}") + response = requests.get(url) + if response.status_code == 200: + soup: BeautifulSoup = BeautifulSoup(response.text, 'html.parser') + return soup + else: + self.logger.error(f"❌ failed to retrieve URL: {url}, Status Code: {response.status_code}") + return None + except Exception as e: + self.logger.error(f"❌ error fetching URL: {url}, Error: {e}") + return None + + def extract_data(self, soup: BeautifulSoup) -> Optional[str]: + elements: List[Tag] = soup.find_all(['p', 'article', 'div']) + paragraphs: List[str] = [] + + for element in elements: + text: str = element.get_text(strip=True) + + if text and text not in paragraphs and len(text) > 10: + paragraphs.append(text) + + formatted_text: str = '\n\n '.join(paragraphs) + return formatted_text diff --git a/src/backend/semantic/lib/spider/spider_selenium.py b/src/backend/semantic/lib/spider/spider_selenium.py new file mode 100644 index 00000000..52024761 --- /dev/null +++ b/src/backend/semantic/lib/spider/spider_selenium.py @@ -0,0 +1,164 @@ +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.chrome.options import Options +from webdriver_manager.chrome import ChromeDriverManager +from bs4 import BeautifulSoup +import time +import logging +from lib.spider.chunked_item import ChunkedItem +from typing import List +from urllib.parse import urljoin, urlparse + +from readiness_probe import ReadinessProbe + + +class SeleniumSpider: + + def __init__(self, base_url): + self.visited = set() + self.collected_data: List[ChunkedItem] = [] + self.base_domain = urlparse(base_url).netloc + self.logger = logging.getLogger(__name__) + + # Initialize WebDriver + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--disable-dev-shm-usage") + self.driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options) + + def fetch_and_parse(self, url): + try: + self.logger.info(f"Processing URL: {url}") + # Open the URL + self.driver.get(url) + # Wait for the dynamic content to load + time.sleep(10) + # Get the page source after all scripts have been executed + html = self.driver.page_source + # Parse the page with BeautifulSoup + soup = BeautifulSoup(html, 'html.parser') + return soup + except Exception as e: + self.logger.error(f"❌ Error fetching URL: {url}, Error: {e}") + return None + + def extract_data(self, soup): + elements = soup.find_all(['p', 'article', 'div']) + paragraphs = [] + + for element in elements: + text = element.get_text(strip=True) + if text: + paragraphs.append(text) + + formatted_text = '\n\n'.join(paragraphs) + return formatted_text + + def process_page(self, url: str, recursive: bool) -> list[ChunkedItem] | None: + start_time = time.time() + + # notifying the readiness probe that the service is alive + ReadinessProbe().update_last_seen() + + if url in self.visited: + return + + self.visited.add(url) + soup = self.fetch_and_parse(url) + if not soup: + return + + page_content = self.extract_data(soup) + if page_content: + self.collected_data.append(ChunkedItem(content=page_content, url=url)) + + links = [a['href'] for a in soup.find_all('a', href=True)] + for link in links: + absolute_link = urljoin(url, link) + parsed_link = urlparse(absolute_link) + if parsed_link.scheme in ['http', 'https'] and absolute_link not in self.visited: + if parsed_link.netloc == self.base_domain: + self.process_page(absolute_link) + + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + self.logger.info(f"Total elapsed time: {elapsed_time:.2f} seconds") + + # Return the collected data only after all recursive calls are complete + return self.collected_data + + def close(self): + self.driver.quit() + +# if __name__ == "__main__": +# logging.basicConfig(level=logging.INFO) +# spider = SeleniumSpider("https://example.com") +# spider.process_page("https://example.com") +# collected_data = spider.get_collected_data() +# for data in collected_data: +# print(data.url) +# print(data.content) +# spider.close() + +# from selenium import webdriver +# from selenium.webdriver.chrome.service import Service +# from selenium.webdriver.chrome.options import Options +# from webdriver_manager.chrome import ChromeDriverManager +# from bs4 import BeautifulSoup +# import time +# import logging +# from lib.chunked_list import ChunkedList +# from typing import List +# from urllib.parse import urljoin, urlparse +# # pip install selenium +# # pip install webdriver-manager +# # pip install beautifulsoup4 + + +# class SeleniumSpider: + +# def __init__(self, base_url): +# self.visited = set() +# self.collected_data: List[ChunkedList] = [] +# self.base_domain = urlparse(base_url).netloc +# self.logger = logging.getLogger(__name__) + +# def get_collected_data(self): +# return self.collected_data + +# def process_page(self, url): +# try: +# self.logger.info("creating webdriver..") + +# # Setup Chrome WebDriver in headless mode +# self.logger.info("Setting up browser options") +# chrome_options = Options() +# chrome_options.add_argument("--headless") # Ensures Chrome runs in headless mode +# chrome_options.add_argument("--no-sandbox") # Bypass OS security model, mandatory on some systems +# chrome_options.add_argument("--disable-dev-shm-usage") # Overcome limited resource problems +# driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options) + +# try: +# # Open the URL +# driver.get(url) + +# # Wait for the dynamic content to load, adjust the wait time as necessary +# time.sleep(10) # Consider using WebDriverWait for a more efficient wait + +# # Get the page source after all scripts have been executed +# html = driver.page_source + +# # Parse the page with BeautifulSoup +# soup = BeautifulSoup(html, 'html.parser') + +# # Extract and print text from each paragraph +# paragraphs = soup.find_all('p') +# for paragraph in paragraphs: +# self.logger.info(paragraph.text) +# finally: +# # Make sure to close the driver +# driver.quit() +# except Exception as e: +# self.logger.exception(e) + diff --git a/src/backend/semantic/readiness_probe.py b/src/backend/semantic/readiness_probe.py new file mode 100644 index 00000000..a50ce31c --- /dev/null +++ b/src/backend/semantic/readiness_probe.py @@ -0,0 +1,73 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer +from datetime import datetime, timedelta +import logging +import os +from dotenv import load_dotenv + +# Define the logging configuration +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +load_dotenv() +readiness_time_out = int(os.getenv('READINESS_TIME_OUT', 500)) + +class ReadinessProbe: + _instance = None + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(ReadinessProbe, cls).__new__(cls, *args, **kwargs) + return cls._instance + + def __init__(self): + if not hasattr(self, 'initialized'): # Ensure the logger is only initialized once + self.logger = logging.getLogger(self.__class__.__name__) + self.last_seen = datetime.utcnow() + self.initialized = True + + def is_service_ready(self): + # Check if the difference between the current time and last_seen is more than the readiness timeout + if datetime.utcnow() - self.last_seen > timedelta(seconds=readiness_time_out): + return False + return True + + def update_last_seen(self): + self.logger.debug("Readiness probe last seen being updated") + self.last_seen = datetime.utcnow() + + class ReadinessProbeHandler(BaseHTTPRequestHandler): + def __init__(self, *args, readiness_probe=None, **kwargs): + self.logger = logging.getLogger(self.__class__.__name__) + self.readiness_probe = readiness_probe + super().__init__(*args, **kwargs) + + def do_GET(self): + if self.path == '/healthz': + if self.readiness_probe.is_service_ready(): + self.send_response(200) + self.end_headers() + self.wfile.write(b"OK") + self.logger.debug("/healthz response 200") + else: + self.send_response(503) + self.end_headers() + self.wfile.write(b"Service Unavailable") + self.logger.error(f"❌ /healthz response 503 - this means the service didn't update_last_seen for " + f"more than {readiness_time_out} seconds ") + else: + self.send_response(404) + self.end_headers() + self.wfile.write(b"Not Found") + self.logger.debug('ReadinessProbeHandler GET /') + + def start_server(self): + try: + self.logger.info("Initializing readiness probe server") + server_address = ('', 8080) + httpd = HTTPServer(server_address, + lambda *args, **kwargs: self.ReadinessProbeHandler(*args, readiness_probe=self, **kwargs)) + self.logger.info(f"Readiness probe server started on {server_address}8080") + httpd.serve_forever() + except Exception as e: + self.logger.error(f"❌ Readiness probe failed to start: {e}") + diff --git a/src/backend/semantic/requirements.txt b/src/backend/semantic/requirements.txt new file mode 100644 index 00000000..82e81922 --- /dev/null +++ b/src/backend/semantic/requirements.txt @@ -0,0 +1,23 @@ +pymilvus==2.4.3 +nats-py==2.7.2 +protobuf==5.27.0 +grpcio==1.63.0 +grpcio-tools==1.63.0 +python-dotenv==1.0.1 +webdriver-manager==4.0.1 +selenium==4.21.0 +beautifulsoup4==4.12.3 +langchain-text-splitters==0.2.0 +PyMuPDF==1.24.4 +pymupdf4llm==0.0.5 +sqlalchemy==2.0.30 +psycopg2-binary==2.9.9 +sqlalchemy-cockroachdb==2.0.2 +youtube-transcript-api==0.6.2 +pymupdf4llm==0.0.5 +PyMuPDFb==1.24.3 +mistune==3.0.2 +minio==7.2.7 + + + diff --git a/src/backend/semantic/semantic-deployment.yaml b/src/backend/semantic/semantic-deployment.yaml new file mode 100644 index 00000000..ec416cf1 --- /dev/null +++ b/src/backend/semantic/semantic-deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: semantic +spec: + replicas: 1 + selector: + matchLabels: + app: semantic + template: + metadata: + labels: + app: semantic + spec: + containers: + - name: semantic + image: cognixacr.azurecr.io/cognix/semantic:main + envFrom: + - configMapRef: + name: semantic-srv + - configMapRef: + name: milvus-cli + - configMapRef: + name: minio-cli + - configMapRef: + name: nats-cli + - configMapRef: + name: embedder-cli + - configMapRef: + name: cockroach-cli + imagePullPolicy: Always + volumeMounts: + - mountPath: /temp + name: temp-volume + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 + failureThreshold: 3 + successThreshold: 1 + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + livenessProbe: + initialDelaySeconds: 60 + timeoutSeconds: 3 + periodSeconds: 60 + failureThreshold: 60 + successThreshold: 1 + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + imagePullSecrets: + - name: pullsecret + volumes: + - name: temp-volume + emptyDir: {} \ No newline at end of file diff --git a/src/backend/semantic/semantic_service.py b/src/backend/semantic/semantic_service.py new file mode 100644 index 00000000..1647b70b --- /dev/null +++ b/src/backend/semantic/semantic_service.py @@ -0,0 +1,185 @@ +#region imports +import asyncio +import logging +import os +import threading +import time +import datetime +from dotenv import load_dotenv +from nats.aio.msg import Msg + +from lib.db.db_connector import ConnectorCRUD, Status +from lib.db.db_document import DocumentCRUD +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.helpers.device_checker import DeviceChecker +from lib.semantic.semantic_factory import SemanticFactory +from lib.db.jetstream_event_subscriber import JetStreamEventSubscriber +from readiness_probe import ReadinessProbe + +#endregion + +#region .env and logs +# Load environment variables from .env file +load_dotenv() + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) + +logger = logging.getLogger(__name__) +logger.info(f"Logging configured with level {log_level_str} and format {log_format}") + +# loading from env env +nats_url = os.getenv('NATS_CLIENT_URL', 'nats://127.0.0.1:4222') +nats_connect_timeout = int(os.getenv('NATS_CLIENT_CONNECT_TIMEOUT', '30')) +nats_reconnect_time_wait = int(os.getenv('NATS_CLIENT_RECONNECT_TIME_WAIT', '30')) +nats_max_reconnect_attempts = int(os.getenv('NATS_CLIENT_MAX_RECONNECT_ATTEMPTS', '3')) +semantic_stream_name = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_NAME', 'semantic') +semantic_stream_subject = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_SUBJECT', 'semantic_activity') +semantic_ack_wait = int(os.getenv('NATS_CLIENT_SEMANTIC_ACK_WAIT', '3600')) # seconds +semantic_max_deliver = int(os.getenv('NATS_CLIENT_SEMANTIC_MAX_DELIVER', '3')) + +cockroach_url = os.getenv('COCKROACH_CLIENT_DATABASE_URL', + 'postgres://root:123@cockroach:26257/defaultdb?sslmode=disable') + + +#endregion + +# Define the event handler function +async def semantic_event(msg: Msg): + start_time = time.time() # Record the start time + connector_id = 0 + entities_analyzed = 0 + try: + logger.info("🔥🔥🔥🔥🔥🔥 starting semantic analysis..") + # Deserialize the message + semantic_data = SemanticData() + semantic_data.ParseFromString(msg.data) + # logger.info(f"message: \n {semantic_data}") + if semantic_data.model_name == "": + logger.error(f"❌ no model name has been passed!") + semantic_data.model_name = "paraphrase-multilingual-mpnet-base-v2" + semantic_data.model_dimension = 768 + + # verify document id is valid otherwise we cannot process the message + if semantic_data.document_id <= 0: + logger.error(f"❌ failed to process semantic data error: document_id must value must be positive") + else: + # see if doc exists + document_crud = DocumentCRUD(cockroach_url) + document = document_crud.select_document(semantic_data.document_id) + + if document: + # needed for th finally block + connector_id = document.connector_id + + # update connector's status + connector_crud = ConnectorCRUD(cockroach_url) + connector = connector_crud.select_connector(document.connector_id) + last_successful_index_date = connector.last_successful_analyzed + + # TODO: IMPORTANT semantic cannot update the status of the connector when it receives + # # multiple messages for the same connector eg, onedrive, teams, web recursive + # because orchestrator or connector they send multiple messages for + # a single connector row + # connector_crud.update_connector(document.connector_id, + # status=Status.PROCESSING, + # last_update=datetime.datetime.utcnow()) + + # performing semantic analysis on the source + semantic = SemanticFactory.create_semantic_analyzer(semantic_data.file_type) + entities_analyzed = await semantic.analyze(data=semantic_data, full_process_start_time=start_time, + ack_wait=semantic_ack_wait, cockroach_url=cockroach_url) + + # if entities_analyzed == 0 this means no data was stored in the vector db + # we shall find a way to tell the user, most likely put the message in the dead letter + + if entities_analyzed is None: + logger.error(f"❌ entities_analyzed is none!!!!!") + entities_analyzed = 0 + # updating again the connector + # TODO: IMPORTANT semantic cannot update the status of the connector when it receives + # # multiple messages for the same connector eg, onedrive, teams, web recursive + # because orchestrator or connector they send multiple messages for + # a single connector row + # connector_crud.update_connector(connector_id, + # status=Status.COMPLETED_SUCCESSFULLY, + # last_successful_analyzed=datetime.datetime.utcnow(), + # last_update=datetime.datetime.utcnow(), + # total_docs_analyzed=entities_analyzed + # # TODO: we are storing the total entities in total docs. one doc will probably generate more than one chunk + # ) + else: + logger.error( + f"❌ failed to process semantic data error: document_id {semantic_data.document_id} not valid") + # Acknowledge the message when done + await msg.ack_sync() + logger.info(f"👍 message acknowledged successfully, total entities stored {entities_analyzed}") + except Exception as e: + error_message = str(e) if e else "Unknown error occurred" + logger.error(f"❌ failed to process semantic data error: {error_message}") + if msg: # Ensure msg is not None before awaiting + await msg.nak() + # try: + # if connector_id != 0: + # connector_crud = ConnectorCRUD(cockroach_url) + # connector_crud.update_connector(connector_id, + # status=Status.COMPLETED_WITH_ERRORS, + # last_update=datetime.datetime.utcnow()) + # except Exception as e: + # error_message = str(e) if e else "Unknown error occurred" + # logger.error(f"❌ failed to process semantic data error: {error_message}") + finally: + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + logger.info(f"⏰⏰ total semantic analysis time: {elapsed_time:.2f} seconds") + + +async def main(): + # Start the readiness probe server in a separate thread + readiness_probe = ReadinessProbe() + readiness_probe_thread = threading.Thread(target=readiness_probe.start_server, daemon=True) + readiness_probe_thread.start() + + # circuit breaker for chunking + # if for reason nats won't be available + # semantic will wait till nats will be up again + while True: + logger.info("🛠️ service starting..") + try: + DeviceChecker.check_device() + # subscribing to jet stream + subscriber = JetStreamEventSubscriber( + nats_url=nats_url, + stream_name=semantic_stream_name, + subject=semantic_stream_subject, + connect_timeout=nats_connect_timeout, + reconnect_time_wait=nats_reconnect_time_wait, + max_reconnect_attempts=nats_max_reconnect_attempts, + ack_wait=semantic_ack_wait, + max_deliver=semantic_max_deliver, + proto_message_type=SemanticData + ) + + subscriber.set_event_handler(semantic_event) + await subscriber.connect_and_subscribe() + + # todo add an event to JetStreamEventSubscriber to signal that connection has been established + logger.info("🚀 service started successfully") + + while True: + await asyncio.sleep(1) + + except KeyboardInterrupt: + logger.info("🛑 Service is stopping due to keyboard interrupt") + except Exception as e: + logger.exception(f"💀 recovering from a fatal error: {e}. The process will restart in 5 seconds..") + await asyncio.sleep(5) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/backend/semantic/test.txt b/src/backend/semantic/test.txt new file mode 100644 index 00000000..c6d6c8e0 --- /dev/null +++ b/src/backend/semantic/test.txt @@ -0,0 +1,2644 @@ +ORCHESTRATOR SEND TO CONNECTOR: +PDF +XPS +TXT +PDF +XPS +DOCX +DOC +XLSX +XLS +PPTX +PPT +HWPX +XPS +EPUB +MOBI +FB2 +CBZ +MD + +ORCHESTRATOR SEND TO SEMANTIC +WEB +YT + +ORCHESTRATOR SEND TO MEDIA +MP4 +MP3 +MPEG +MPGA +M4A +WAV +WEBM +MOV + +2024-06-11T19:07:42.294775085Z 2024-06-11 19:07:42,294 - __main__ - INFO - Logging configured with level INFO and format %(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s +2024-06-11T19:07:42.295228169Z 2024-06-11 19:07:42,295 - ReadinessProbe - INFO - Initializing readiness probe server +2024-06-11T19:07:42.295234002Z 2024-06-11 19:07:42,295 - __main__ - INFO - 🛠️ service starting.. +2024-06-11T19:07:42.295625835Z 2024-06-11 19:07:42,295 - JetStreamEventSubscriber - INFO - 🔌 connecting to nats endpoint nats://nats:4222 .. +2024-06-11T19:07:42.296227627Z 2024-06-11 19:07:42,296 - ReadinessProbe - INFO - Readiness probe server started on ('', 8080)8080 +2024-06-11T19:07:42.297687377Z 2024-06-11 19:07:42,297 - JetStreamEventSubscriber - INFO - successfully connected nats://nats:4222 +2024-06-11T19:07:42.298719710Z 2024-06-11 19:07:42,298 - JetStreamEventSubscriber - INFO - subscribing to jetstream semantic - semantic_activity .. +2024-06-11T19:07:42.299323294Z 2024-06-11 19:07:42,299 - JetStreamEventSubscriber - INFO - successfully subscribed to jetstream semantic - semantic_activity +2024-06-11T19:07:42.305397419Z 2024-06-11 19:07:42,304 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.763.763.1718132806214861809.245', data=b'\n=https://help.collaboard.app/_hcms/mem/reset/request?hsLang=en(\x81\x80\xcc\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:07:42.305414835Z 2024-06-11 19:07:42,304 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:07:42.305532919Z 2024-06-11 19:07:42,305 - __main__ - INFO - message: +2024-06-11T19:07:42.305537252Z url: "https://help.collaboard.app/_hcms/mem/reset/request?hsLang=en" +2024-06-11T19:07:42.305538710Z document_id: 976690891543740417 +2024-06-11T19:07:42.305539960Z connector_id: 976341399530373121 +2024-06-11T19:07:42.305541044Z file_type: URL +2024-06-11T19:07:42.305542169Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:07:42.305543460Z +2024-06-11T19:07:42.305544585Z 2024-06-11 19:07:42,305 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:07:42.305545835Z 2024-06-11 19:07:42,305 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:07:42.402090919Z 2024-06-11 19:07:42,401 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:07:42.402109252Z 2024-06-11 19:07:42,401 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/_hcms/mem/reset/request?hsLang=en +2024-06-11T19:07:42.402111044Z 2024-06-11 19:07:42,401 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/_hcms/mem/reset/request?hsLang=en +2024-06-11T19:07:42.724851710Z 2024-06-11 19:07:42,724 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.32 seconds +2024-06-11T19:07:42.741090919Z 2024-06-11 19:07:42,740 - Milvus_DB - INFO - deleting all entities related to document 976690891543740417 +2024-06-11T19:07:42.778335544Z 2024-06-11 19:07:42,778 - Milvus_DB - INFO - ⏰ total elapsed time: 0.04 seconds +2024-06-11T19:07:42.791310086Z 2024-06-11 19:07:42,790 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:07:42.791318377Z 2024-06-11 19:07:42,790 - root - INFO - splitting the entity +2024-06-11T19:07:42.791319919Z 2024-06-11 19:07:42,790 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:07:42.791321252Z 2024-06-11 19:07:42,791 - URLSemantic - INFO - created 2 chunks for https://help.collaboard.app/_hcms/mem/reset/request?hsLang=en +2024-06-11T19:07:42.791322336Z 2024-06-11 19:07:42,791 - root - INFO - saving in milvus 2 chunk(s) +2024-06-11T19:07:43.402667336Z 2024-06-11 19:07:43,402 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:07:46.452302546Z 2024-06-11 19:07:46,451 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:07:46.459675837Z 2024-06-11 19:07:46,459 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132865.326126378,0): "sql txn" meta={id=6f52409e key=/Table/121/1/976691090074664961/0 iso=Serializable pri=0.00204523 epo=0 ts=1718132865.485210212,1 min=1718132862.797033711,0 seq=1} lock=true stat=PENDING rts=1718132862.797033711,0 wto=false gul=1718132863.297033711,0 +2024-06-11T19:07:46.459685629Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:07:46.459687129Z +2024-06-11T19:07:46.459688421Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:07:46.459689462Z 2024-06-11 19:07:46,459 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:07:46.466609421Z 2024-06-11 19:07:46,466 - __main__ - INFO - 1 +2024-06-11T19:07:46.468095504Z 2024-06-11 19:07:46,467 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:07:46.468099046Z 2024-06-11 19:07:46,467 - __main__ - INFO - ⏰⏰ total elapsed time: 4.16 seconds +2024-06-11T19:07:46.468100421Z 2024-06-11 19:07:46,467 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.763.763.1718132806214861809.245', data=b'\n=https://help.collaboard.app/_hcms/mem/reset/request?hsLang=en(\x81\x80\xcc\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:07:46.468919796Z 2024-06-11 19:07:46,468 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.765.765.1718132806217402684.243', data=b'\n+https://help.collaboard.app/getting-started(\x81\x80\xd0\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:07:46.468925296Z 2024-06-11 19:07:46,468 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:07:46.468933296Z 2024-06-11 19:07:46,468 - __main__ - INFO - message: +2024-06-11T19:07:46.468934587Z url: "https://help.collaboard.app/getting-started" +2024-06-11T19:07:46.468936379Z document_id: 976690891543805953 +2024-06-11T19:07:46.468937296Z connector_id: 976341399530373121 +2024-06-11T19:07:46.468938171Z file_type: URL +2024-06-11T19:07:46.468939046Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:07:46.468940087Z +2024-06-11T19:07:46.468940962Z 2024-06-11 19:07:46,468 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:07:46.469538671Z 2024-06-11 19:07:46,468 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:07:46.575307712Z 2024-06-11 19:07:46,575 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:07:46.575321504Z 2024-06-11 19:07:46,575 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/getting-started +2024-06-11T19:07:46.575323004Z 2024-06-11 19:07:46,575 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/getting-started +2024-06-11T19:07:46.757693754Z 2024-06-11 19:07:46,757 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.18 seconds +2024-06-11T19:07:46.779729004Z 2024-06-11 19:07:46,779 - Milvus_DB - INFO - deleting all entities related to document 976690891543805953 +2024-06-11T19:07:46.794555046Z 2024-06-11 19:07:46,793 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:07:46.811247712Z 2024-06-11 19:07:46,810 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:07:46.811256379Z 2024-06-11 19:07:46,810 - root - INFO - splitting the entity +2024-06-11T19:07:46.811257754Z 2024-06-11 19:07:46,810 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:07:46.811259129Z 2024-06-11 19:07:46,810 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/getting-started +2024-06-11T19:07:46.811260171Z 2024-06-11 19:07:46,810 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:07:47.379714921Z 2024-06-11 19:07:47,379 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:07:50.488415714Z 2024-06-11 19:07:50,487 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:07:50.494979964Z 2024-06-11 19:07:50,494 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132869.370063589,0): "sql txn" meta={id=ff4404ff key=/Table/121/1/976691103299764225/0 iso=Serializable pri=0.00054259 epo=0 ts=1718132869.519285839,1 min=1718132866.816276379,0 seq=1} lock=true stat=PENDING rts=1718132866.816276379,0 wto=false gul=1718132867.316276379,0 +2024-06-11T19:07:50.494988756Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:07:50.494996839Z +2024-06-11T19:07:50.494998131Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:07:50.494999173Z 2024-06-11 19:07:50,494 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:07:50.500317881Z 2024-06-11 19:07:50,500 - __main__ - INFO - 1 +2024-06-11T19:07:50.501178381Z 2024-06-11 19:07:50,500 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:07:50.501344339Z 2024-06-11 19:07:50,501 - __main__ - INFO - ⏰⏰ total elapsed time: 4.03 seconds +2024-06-11T19:07:50.501684131Z 2024-06-11 19:07:50,501 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.765.765.1718132806217402684.243', data=b'\n+https://help.collaboard.app/getting-started(\x81\x80\xd0\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:07:50.502906464Z 2024-06-11 19:07:50,502 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.767.767.1718132806220214934.241', data=b'\n=https://help.collaboard.app/_hcms/mem/login?redirect_url=/de/(\x81\x80\xd4\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:07:50.502910339Z 2024-06-11 19:07:50,502 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:07:50.502911714Z 2024-06-11 19:07:50,502 - __main__ - INFO - message: +2024-06-11T19:07:50.502912756Z url: "https://help.collaboard.app/_hcms/mem/login?redirect_url=/de/" +2024-06-11T19:07:50.502913798Z document_id: 976690891543871489 +2024-06-11T19:07:50.502915423Z connector_id: 976341399530373121 +2024-06-11T19:07:50.502916339Z file_type: URL +2024-06-11T19:07:50.502917256Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:07:50.502918256Z +2024-06-11T19:07:50.502919173Z 2024-06-11 19:07:50,502 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:07:50.502920256Z 2024-06-11 19:07:50,502 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:07:50.546076631Z 2024-06-11 19:07:50,545 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:07:50.546087381Z 2024-06-11 19:07:50,545 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/_hcms/mem/login?redirect_url=/de/ +2024-06-11T19:07:50.546088881Z 2024-06-11 19:07:50,545 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/_hcms/mem/login?redirect_url=/de/ +2024-06-11T19:07:50.641240423Z 2024-06-11 19:07:50,641 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:07:50.656423673Z 2024-06-11 19:07:50,656 - Milvus_DB - INFO - deleting all entities related to document 976690891543871489 +2024-06-11T19:07:50.668011756Z 2024-06-11 19:07:50,667 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:07:50.678716464Z 2024-06-11 19:07:50,678 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:07:50.678725839Z 2024-06-11 19:07:50,678 - root - INFO - splitting the entity +2024-06-11T19:07:50.678727256Z 2024-06-11 19:07:50,678 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:07:50.678782048Z 2024-06-11 19:07:50,678 - URLSemantic - INFO - created 2 chunks for https://help.collaboard.app/_hcms/mem/login?redirect_url=/de/ +2024-06-11T19:07:50.678785131Z 2024-06-11 19:07:50,678 - root - INFO - saving in milvus 2 chunk(s) +2024-06-11T19:07:51.314475256Z 2024-06-11 19:07:51,314 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:07:53.853991882Z 2024-06-11 19:07:53,853 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.17 seconds +2024-06-11T19:07:53.859368716Z 2024-06-11 19:07:53,859 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132873.358645091,0): "sql txn" meta={id=5f0ad0a0 key=/Table/121/1/976691114327343105/0 iso=Serializable pri=0.00630096 epo=0 ts=1718132873.606796091,1 min=1718132870.683411381,0 seq=1} lock=true stat=PENDING rts=1718132870.683411381,0 wto=false gul=1718132871.183411381,0 +2024-06-11T19:07:53.859407632Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:07:53.859412591Z +2024-06-11T19:07:53.859413841Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:07:53.859415049Z 2024-06-11 19:07:53,859 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:07:53.864915507Z 2024-06-11 19:07:53,864 - __main__ - INFO - 1 +2024-06-11T19:07:53.865445591Z 2024-06-11 19:07:53,865 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:07:53.865456382Z 2024-06-11 19:07:53,865 - __main__ - INFO - ⏰⏰ total elapsed time: 3.36 seconds +2024-06-11T19:07:53.865458049Z 2024-06-11 19:07:53,865 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.767.767.1718132806220214934.241', data=b'\n=https://help.collaboard.app/_hcms/mem/login?redirect_url=/de/(\x81\x80\xd4\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:07:53.866925382Z 2024-06-11 19:07:53,866 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.769.769.1718132806223042934.239', data=b'\n,https://help.collaboard.app/de/abos-rechnung(\x81\x80\xd8\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:07:53.866936424Z 2024-06-11 19:07:53,866 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:07:53.866937882Z 2024-06-11 19:07:53,866 - __main__ - INFO - message: +2024-06-11T19:07:53.866938966Z url: "https://help.collaboard.app/de/abos-rechnung" +2024-06-11T19:07:53.866940091Z document_id: 976690891543937025 +2024-06-11T19:07:53.866940966Z connector_id: 976341399530373121 +2024-06-11T19:07:53.866941882Z file_type: URL +2024-06-11T19:07:53.866942799Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:07:53.866943924Z +2024-06-11T19:07:53.866945091Z 2024-06-11 19:07:53,866 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:07:53.867049507Z 2024-06-11 19:07:53,866 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:07:53.898595341Z 2024-06-11 19:07:53,898 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:07:53.898612591Z 2024-06-11 19:07:53,898 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/abos-rechnung +2024-06-11T19:07:53.898614174Z 2024-06-11 19:07:53,898 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/abos-rechnung +2024-06-11T19:07:54.112481383Z 2024-06-11 19:07:54,112 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.21 seconds +2024-06-11T19:07:54.133359716Z 2024-06-11 19:07:54,133 - Milvus_DB - INFO - deleting all entities related to document 976690891543937025 +2024-06-11T19:07:54.148226424Z 2024-06-11 19:07:54,148 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:07:54.164676674Z 2024-06-11 19:07:54,163 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:07:54.164682966Z 2024-06-11 19:07:54,163 - root - INFO - splitting the entity +2024-06-11T19:07:54.164684133Z 2024-06-11 19:07:54,163 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:07:54.164685883Z 2024-06-11 19:07:54,163 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/abos-rechnung +2024-06-11T19:07:54.164686966Z 2024-06-11 19:07:54,163 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:07:54.721388508Z 2024-06-11 19:07:54,721 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:07:58.276164718Z 2024-06-11 19:07:58,275 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.10 seconds +2024-06-11T19:07:58.282182176Z 2024-06-11 19:07:58,282 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132876.905362926,0): "sql txn" meta={id=fe03fe15 key=/Table/121/1/976691128819712001/0 iso=Serializable pri=0.01358539 epo=0 ts=1718132877.052638467,1 min=1718132874.170719924,0 seq=1} lock=true stat=PENDING rts=1718132874.170719924,0 wto=false gul=1718132874.670719924,0 +2024-06-11T19:07:58.282214760Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:07:58.282217801Z +2024-06-11T19:07:58.282219135Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:07:58.282220426Z 2024-06-11 19:07:58,282 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:07:58.288838260Z 2024-06-11 19:07:58,288 - __main__ - INFO - 1 +2024-06-11T19:07:58.290268760Z 2024-06-11 19:07:58,289 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:07:58.290273593Z 2024-06-11 19:07:58,289 - __main__ - INFO - ⏰⏰ total elapsed time: 4.42 seconds +2024-06-11T19:07:58.290275176Z 2024-06-11 19:07:58,289 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.769.769.1718132806223042934.239', data=b'\n,https://help.collaboard.app/de/abos-rechnung(\x81\x80\xd8\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:07:58.291274051Z 2024-06-11 19:07:58,290 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.771.771.1718132806226899559.237', data=b'\n9https://help.collaboard.app/de/technische-fragen-probleme(\x81\x80\xdc\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:07:58.291279676Z 2024-06-11 19:07:58,290 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:07:58.291281593Z 2024-06-11 19:07:58,290 - __main__ - INFO - message: +2024-06-11T19:07:58.291282801Z url: "https://help.collaboard.app/de/technische-fragen-probleme" +2024-06-11T19:07:58.291284301Z document_id: 976690891544002561 +2024-06-11T19:07:58.291288468Z connector_id: 976341399530373121 +2024-06-11T19:07:58.291289676Z file_type: URL +2024-06-11T19:07:58.291290760Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:07:58.291291926Z +2024-06-11T19:07:58.291292968Z 2024-06-11 19:07:58,290 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:07:58.291294260Z 2024-06-11 19:07:58,290 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:07:58.326447718Z 2024-06-11 19:07:58,326 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:07:58.326459510Z 2024-06-11 19:07:58,326 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/technische-fragen-probleme +2024-06-11T19:07:58.326461260Z 2024-06-11 19:07:58,326 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/technische-fragen-probleme +2024-06-11T19:07:58.435811468Z 2024-06-11 19:07:58,435 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:07:58.448932051Z 2024-06-11 19:07:58,448 - Milvus_DB - INFO - deleting all entities related to document 976690891544002561 +2024-06-11T19:07:58.461523676Z 2024-06-11 19:07:58,461 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:07:58.472095926Z 2024-06-11 19:07:58,471 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:07:58.472102801Z 2024-06-11 19:07:58,472 - root - INFO - splitting the entity +2024-06-11T19:07:58.472104468Z 2024-06-11 19:07:58,472 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:07:58.472526301Z 2024-06-11 19:07:58,472 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/de/technische-fragen-probleme +2024-06-11T19:07:58.472529676Z 2024-06-11 19:07:58,472 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:07:59.059274468Z 2024-06-11 19:07:59,059 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:08:02.095260095Z 2024-06-11 19:08:02,095 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:08:02.100732303Z 2024-06-11 19:08:02,100 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132880.317051427,0): "sql txn" meta={id=9c9c0a0a key=/Table/121/1/976691141333286913/0 iso=Serializable pri=0.02836636 epo=0 ts=1718132880.470663219,1 min=1718132878.478481343,0 seq=1} lock=true stat=PENDING rts=1718132878.478481343,0 wto=false gul=1718132878.978481343,0 +2024-06-11T19:08:02.100744720Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:02.100746053Z +2024-06-11T19:08:02.100747178Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:02.100748261Z 2024-06-11 19:08:02,100 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:02.105892928Z 2024-06-11 19:08:02,105 - __main__ - INFO - 1 +2024-06-11T19:08:02.106917803Z 2024-06-11 19:08:02,106 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:02.106923136Z 2024-06-11 19:08:02,106 - __main__ - INFO - ⏰⏰ total elapsed time: 3.82 seconds +2024-06-11T19:08:02.106964845Z 2024-06-11 19:08:02,106 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.771.771.1718132806226899559.237', data=b'\n9https://help.collaboard.app/de/technische-fragen-probleme(\x81\x80\xdc\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:02.107668678Z 2024-06-11 19:08:02,107 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.773.773.1718132806230178434.235', data=b'\n3https://help.collaboard.app/working-with-collaboard(\x81\x80\xe0\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:02.107679178Z 2024-06-11 19:08:02,107 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:02.107680636Z 2024-06-11 19:08:02,107 - __main__ - INFO - message: +2024-06-11T19:08:02.107681761Z url: "https://help.collaboard.app/working-with-collaboard" +2024-06-11T19:08:02.107682803Z document_id: 976690891544068097 +2024-06-11T19:08:02.107683678Z connector_id: 976341399530373121 +2024-06-11T19:08:02.107684928Z file_type: URL +2024-06-11T19:08:02.107685845Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:02.107686845Z +2024-06-11T19:08:02.107687720Z 2024-06-11 19:08:02,107 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:02.107688761Z 2024-06-11 19:08:02,107 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:02.138521678Z 2024-06-11 19:08:02,138 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:02.138534595Z 2024-06-11 19:08:02,138 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/working-with-collaboard +2024-06-11T19:08:02.138608136Z 2024-06-11 19:08:02,138 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/working-with-collaboard +2024-06-11T19:08:02.259241345Z 2024-06-11 19:08:02,258 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:08:02.273258511Z 2024-06-11 19:08:02,273 - Milvus_DB - INFO - deleting all entities related to document 976690891544068097 +2024-06-11T19:08:02.285370886Z 2024-06-11 19:08:02,285 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:02.295675803Z 2024-06-11 19:08:02,295 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:02.295687636Z 2024-06-11 19:08:02,295 - root - INFO - splitting the entity +2024-06-11T19:08:02.295689095Z 2024-06-11 19:08:02,295 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:02.295990303Z 2024-06-11 19:08:02,295 - URLSemantic - INFO - created 13 chunks for https://help.collaboard.app/working-with-collaboard +2024-06-11T19:08:02.295994345Z 2024-06-11 19:08:02,295 - root - INFO - saving in milvus 13 chunk(s) +2024-06-11T19:08:02.852343053Z 2024-06-11 19:08:02,852 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:08:05.896518972Z 2024-06-11 19:08:05,896 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.59 seconds +2024-06-11T19:08:05.901724638Z 2024-06-11 19:08:05,901 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132883.939758596,0): "sql txn" meta={id=fc91bafe key=/Table/121/1/976691153788960769/0 iso=Serializable pri=0.01628366 epo=0 ts=1718132884.122800887,1 min=1718132882.300473470,0 seq=1} lock=true stat=PENDING rts=1718132882.300473470,0 wto=false gul=1718132882.800473470,0 +2024-06-11T19:08:05.901736680Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:05.901738138Z +2024-06-11T19:08:05.901739347Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:05.901740555Z 2024-06-11 19:08:05,901 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:05.906515472Z 2024-06-11 19:08:05,906 - __main__ - INFO - 1 +2024-06-11T19:08:05.907992847Z 2024-06-11 19:08:05,907 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:05.908001722Z 2024-06-11 19:08:05,907 - __main__ - INFO - ⏰⏰ total elapsed time: 3.80 seconds +2024-06-11T19:08:05.908003222Z 2024-06-11 19:08:05,907 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.773.773.1718132806230178434.235', data=b'\n3https://help.collaboard.app/working-with-collaboard(\x81\x80\xe0\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:05.916139722Z 2024-06-11 19:08:05,915 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.775.775.1718132806232954100.233', data=b'\n*https://help.collaboard.app/administration(\x81\x80\xe4\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:05.916146972Z 2024-06-11 19:08:05,915 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:05.916148430Z 2024-06-11 19:08:05,916 - __main__ - INFO - message: +2024-06-11T19:08:05.916149430Z url: "https://help.collaboard.app/administration" +2024-06-11T19:08:05.916150513Z document_id: 976690891544133633 +2024-06-11T19:08:05.916151388Z connector_id: 976341399530373121 +2024-06-11T19:08:05.916152305Z file_type: URL +2024-06-11T19:08:05.916153138Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:05.916154222Z +2024-06-11T19:08:05.916156763Z 2024-06-11 19:08:05,916 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:05.916157805Z 2024-06-11 19:08:05,916 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:05.956497847Z 2024-06-11 19:08:05,956 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:05.956516305Z 2024-06-11 19:08:05,956 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/administration +2024-06-11T19:08:05.956518055Z 2024-06-11 19:08:05,956 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/administration +2024-06-11T19:08:06.050680972Z 2024-06-11 19:08:06,050 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.09 seconds +2024-06-11T19:08:06.063493680Z 2024-06-11 19:08:06,063 - Milvus_DB - INFO - deleting all entities related to document 976690891544133633 +2024-06-11T19:08:06.076558222Z 2024-06-11 19:08:06,076 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:06.086056055Z 2024-06-11 19:08:06,085 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:06.086063847Z 2024-06-11 19:08:06,085 - root - INFO - splitting the entity +2024-06-11T19:08:06.086065597Z 2024-06-11 19:08:06,085 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:06.086358055Z 2024-06-11 19:08:06,086 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/administration +2024-06-11T19:08:06.086363388Z 2024-06-11 19:08:06,086 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:08:06.643268264Z 2024-06-11 19:08:06,643 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:08:09.193040098Z 2024-06-11 19:08:09,192 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.10 seconds +2024-06-11T19:08:09.197755473Z 2024-06-11 19:08:09,197 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132887.915728556,0): "sql txn" meta={id=825ef954 key=/Table/121/1/976691164590178305/0 iso=Serializable pri=0.00332785 epo=0 ts=1718132888.200122264,1 min=1718132886.090540722,0 seq=1} lock=true stat=PENDING rts=1718132886.090540722,0 wto=false gul=1718132886.590540722,0 +2024-06-11T19:08:09.197763848Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:09.197765598Z +2024-06-11T19:08:09.197766931Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:09.197768181Z 2024-06-11 19:08:09,197 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:09.202306973Z 2024-06-11 19:08:09,201 - __main__ - INFO - 1 +2024-06-11T19:08:09.203368265Z 2024-06-11 19:08:09,202 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:09.203375806Z 2024-06-11 19:08:09,203 - __main__ - INFO - ⏰⏰ total elapsed time: 3.29 seconds +2024-06-11T19:08:09.203377598Z 2024-06-11 19:08:09,203 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.775.775.1718132806232954100.233', data=b'\n*https://help.collaboard.app/administration(\x81\x80\xe4\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:09.204381348Z 2024-06-11 19:08:09,203 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.777.777.1718132806235849225.231', data=b'\n3https://help.collaboard.app/de/support-kontaktieren(\x81\x80\xe8\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:09.204393015Z 2024-06-11 19:08:09,203 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:09.204394473Z 2024-06-11 19:08:09,203 - __main__ - INFO - message: +2024-06-11T19:08:09.204395598Z url: "https://help.collaboard.app/de/support-kontaktieren" +2024-06-11T19:08:09.204396723Z document_id: 976690891544199169 +2024-06-11T19:08:09.204397681Z connector_id: 976341399530373121 +2024-06-11T19:08:09.204398640Z file_type: URL +2024-06-11T19:08:09.204399556Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:09.204400681Z +2024-06-11T19:08:09.204401598Z 2024-06-11 19:08:09,203 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:09.204402681Z 2024-06-11 19:08:09,203 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:09.236884556Z 2024-06-11 19:08:09,236 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:09.236893015Z 2024-06-11 19:08:09,236 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/support-kontaktieren +2024-06-11T19:08:09.236894515Z 2024-06-11 19:08:09,236 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/support-kontaktieren +2024-06-11T19:08:09.384021307Z 2024-06-11 19:08:09,383 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.15 seconds +2024-06-11T19:08:09.399046140Z 2024-06-11 19:08:09,398 - Milvus_DB - INFO - deleting all entities related to document 976690891544199169 +2024-06-11T19:08:09.410655682Z 2024-06-11 19:08:09,410 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:09.420784848Z 2024-06-11 19:08:09,420 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:09.420818848Z 2024-06-11 19:08:09,420 - root - INFO - splitting the entity +2024-06-11T19:08:09.420820932Z 2024-06-11 19:08:09,420 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:09.421008557Z 2024-06-11 19:08:09,420 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/de/support-kontaktieren +2024-06-11T19:08:09.421011432Z 2024-06-11 19:08:09,420 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:08:10.022801099Z 2024-06-11 19:08:10,022 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:08:13.051726225Z 2024-06-11 19:08:13,051 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:08:13.057580267Z 2024-06-11 19:08:13,057 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132892.015887558,0): "sql txn" meta={id=f23ede02 key=/Table/121/1/976691177234989057/0 iso=Serializable pri=0.01715957 epo=0 ts=1718132892.191264308,1 min=1718132889.427401015,0 seq=1} lock=true stat=PENDING rts=1718132889.427401015,0 wto=false gul=1718132889.927401015,0 +2024-06-11T19:08:13.057593767Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:13.057595725Z +2024-06-11T19:08:13.057596933Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:13.057598350Z 2024-06-11 19:08:13,057 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:13.062094392Z 2024-06-11 19:08:13,061 - __main__ - INFO - 1 +2024-06-11T19:08:13.063302350Z 2024-06-11 19:08:13,062 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:13.063306517Z 2024-06-11 19:08:13,063 - __main__ - INFO - ⏰⏰ total elapsed time: 3.86 seconds +2024-06-11T19:08:13.063646558Z 2024-06-11 19:08:13,063 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.777.777.1718132806235849225.231', data=b'\n3https://help.collaboard.app/de/support-kontaktieren(\x81\x80\xe8\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:13.064121350Z 2024-06-11 19:08:13,063 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.779.779.1718132806238435642.229', data=b'\n?https://help.collaboard.app/de/anfragefehler-401-beim-einloggen(\x81\x80\xec\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:13.064126433Z 2024-06-11 19:08:13,063 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:13.064127975Z 2024-06-11 19:08:13,063 - __main__ - INFO - message: +2024-06-11T19:08:13.064129100Z url: "https://help.collaboard.app/de/anfragefehler-401-beim-einloggen" +2024-06-11T19:08:13.064130267Z document_id: 976690891544264705 +2024-06-11T19:08:13.064131225Z connector_id: 976341399530373121 +2024-06-11T19:08:13.064132142Z file_type: URL +2024-06-11T19:08:13.064133017Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:13.064134058Z +2024-06-11T19:08:13.064134892Z 2024-06-11 19:08:13,063 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:13.064135933Z 2024-06-11 19:08:13,063 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:13.098475642Z 2024-06-11 19:08:13,098 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:13.098490767Z 2024-06-11 19:08:13,098 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/anfragefehler-401-beim-einloggen +2024-06-11T19:08:13.098492475Z 2024-06-11 19:08:13,098 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/anfragefehler-401-beim-einloggen +2024-06-11T19:08:13.297937808Z 2024-06-11 19:08:13,297 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.20 seconds +2024-06-11T19:08:13.316241350Z 2024-06-11 19:08:13,315 - Milvus_DB - INFO - deleting all entities related to document 976690891544264705 +2024-06-11T19:08:13.330174683Z 2024-06-11 19:08:13,330 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:13.341269433Z 2024-06-11 19:08:13,341 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:13.341288475Z 2024-06-11 19:08:13,341 - root - INFO - splitting the entity +2024-06-11T19:08:13.341290058Z 2024-06-11 19:08:13,341 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:13.341731475Z 2024-06-11 19:08:13,341 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/de/anfragefehler-401-beim-einloggen +2024-06-11T19:08:13.341737808Z 2024-06-11 19:08:13,341 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:08:13.924630392Z 2024-06-11 19:08:13,923 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:08:17.010543685Z 2024-06-11 19:08:17,010 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:08:17.019654727Z 2024-06-11 19:08:17,019 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132895.942278185,0): "sql txn" meta={id=17fb111d key=/Table/121/1/976691190207315969/0 iso=Serializable pri=0.00578105 epo=0 ts=1718132896.080102393,1 min=1718132893.346782058,0 seq=1} lock=true stat=PENDING rts=1718132893.346782058,0 wto=false gul=1718132893.846782058,0 +2024-06-11T19:08:17.019668935Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:17.019670435Z +2024-06-11T19:08:17.019671435Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:17.019672560Z 2024-06-11 19:08:17,019 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:17.030848394Z 2024-06-11 19:08:17,030 - __main__ - INFO - 1 +2024-06-11T19:08:17.032139102Z 2024-06-11 19:08:17,031 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:17.032143019Z 2024-06-11 19:08:17,031 - __main__ - INFO - ⏰⏰ total elapsed time: 3.97 seconds +2024-06-11T19:08:17.032144269Z 2024-06-11 19:08:17,031 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.779.779.1718132806238435642.229', data=b'\n?https://help.collaboard.app/de/anfragefehler-401-beim-einloggen(\x81\x80\xec\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:17.033342977Z 2024-06-11 19:08:17,032 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.781.781.1718132806240632434.227', data=b'\nFhttps://help.collaboard.app/de/ich-habe-meine-e-mail-adresse-vergessen(\x81\x80\xf0\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:17.033353269Z 2024-06-11 19:08:17,032 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:17.033355102Z 2024-06-11 19:08:17,032 - __main__ - INFO - message: +2024-06-11T19:08:17.033356352Z url: "https://help.collaboard.app/de/ich-habe-meine-e-mail-adresse-vergessen" +2024-06-11T19:08:17.033357560Z document_id: 976690891544330241 +2024-06-11T19:08:17.033358519Z connector_id: 976341399530373121 +2024-06-11T19:08:17.033359394Z file_type: URL +2024-06-11T19:08:17.033360310Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:17.033361477Z +2024-06-11T19:08:17.033362352Z 2024-06-11 19:08:17,032 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:17.033363394Z 2024-06-11 19:08:17,032 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:17.071566477Z 2024-06-11 19:08:17,071 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:17.071583102Z 2024-06-11 19:08:17,071 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/ich-habe-meine-e-mail-adresse-vergessen +2024-06-11T19:08:17.071584602Z 2024-06-11 19:08:17,071 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/ich-habe-meine-e-mail-adresse-vergessen +2024-06-11T19:08:17.186176477Z 2024-06-11 19:08:17,185 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:08:17.197945019Z 2024-06-11 19:08:17,197 - Milvus_DB - INFO - deleting all entities related to document 976690891544330241 +2024-06-11T19:08:17.209127685Z 2024-06-11 19:08:17,209 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:17.217280894Z 2024-06-11 19:08:17,217 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:17.217287144Z 2024-06-11 19:08:17,217 - root - INFO - splitting the entity +2024-06-11T19:08:17.217288519Z 2024-06-11 19:08:17,217 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:17.217637935Z 2024-06-11 19:08:17,217 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/de/ich-habe-meine-e-mail-adresse-vergessen +2024-06-11T19:08:17.217642560Z 2024-06-11 19:08:17,217 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:08:17.807954394Z 2024-06-11 19:08:17,807 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:08:20.887153729Z 2024-06-11 19:08:20,886 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:08:20.893557270Z 2024-06-11 19:08:20,893 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132899.375610061,0): "sql txn" meta={id=b8a7bdb4 key=/Table/121/1/976691202909896705/0 iso=Serializable pri=0.04631584 epo=0 ts=1718132899.563336603,1 min=1718132897.221609810,0 seq=1} lock=true stat=PENDING rts=1718132897.221609810,0 wto=false gul=1718132897.721609810,0 +2024-06-11T19:08:20.893569229Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:20.893571062Z +2024-06-11T19:08:20.893572270Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:20.893573520Z 2024-06-11 19:08:20,893 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:20.898171895Z 2024-06-11 19:08:20,897 - __main__ - INFO - 1 +2024-06-11T19:08:20.899766854Z 2024-06-11 19:08:20,899 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:20.899770854Z 2024-06-11 19:08:20,899 - __main__ - INFO - ⏰⏰ total elapsed time: 3.87 seconds +2024-06-11T19:08:20.899772270Z 2024-06-11 19:08:20,899 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.781.781.1718132806240632434.227', data=b'\nFhttps://help.collaboard.app/de/ich-habe-meine-e-mail-adresse-vergessen(\x81\x80\xf0\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:20.901467395Z 2024-06-11 19:08:20,900 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.783.783.1718132806243058309.225', data=b'\n2https://help.collaboard.app/de/support-kontakieren(\x81\x80\xf4\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:20.901517979Z 2024-06-11 19:08:20,900 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:20.901528145Z 2024-06-11 19:08:20,900 - __main__ - INFO - message: +2024-06-11T19:08:20.901529854Z url: "https://help.collaboard.app/de/support-kontakieren" +2024-06-11T19:08:20.901531104Z document_id: 976690891544395777 +2024-06-11T19:08:20.901532145Z connector_id: 976341399530373121 +2024-06-11T19:08:20.901533145Z file_type: URL +2024-06-11T19:08:20.901534104Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:20.901535187Z +2024-06-11T19:08:20.901536104Z 2024-06-11 19:08:20,900 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:20.901537187Z 2024-06-11 19:08:20,900 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:20.936930229Z 2024-06-11 19:08:20,936 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:20.936946395Z 2024-06-11 19:08:20,936 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/support-kontakieren +2024-06-11T19:08:20.936948270Z 2024-06-11 19:08:20,936 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/support-kontakieren +2024-06-11T19:08:21.034092729Z 2024-06-11 19:08:21,033 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:08:21.046350562Z 2024-06-11 19:08:21,046 - Milvus_DB - INFO - deleting all entities related to document 976690891544395777 +2024-06-11T19:08:21.056492395Z 2024-06-11 19:08:21,056 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:21.066004104Z 2024-06-11 19:08:21,065 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:21.066013270Z 2024-06-11 19:08:21,065 - root - INFO - splitting the entity +2024-06-11T19:08:21.066014729Z 2024-06-11 19:08:21,065 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:21.066258979Z 2024-06-11 19:08:21,066 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/support-kontakieren +2024-06-11T19:08:21.066264354Z 2024-06-11 19:08:21,066 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:08:21.622689062Z 2024-06-11 19:08:21,622 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:08:24.190474147Z 2024-06-11 19:08:24,190 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.12 seconds +2024-06-11T19:08:24.195983355Z 2024-06-11 19:08:24,195 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132902.930615063,0): "sql txn" meta={id=5d87d51f key=/Table/121/1/976691213734248449/0 iso=Serializable pri=0.04169350 epo=0 ts=1718132903.095081313,1 min=1718132901.070092312,0 seq=1} lock=true stat=PENDING rts=1718132901.070092312,0 wto=false gul=1718132901.570092312,0 +2024-06-11T19:08:24.195994064Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:24.195995730Z +2024-06-11T19:08:24.195996814Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:24.195997897Z 2024-06-11 19:08:24,195 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:24.201444105Z 2024-06-11 19:08:24,201 - __main__ - INFO - 1 +2024-06-11T19:08:24.202901439Z 2024-06-11 19:08:24,202 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:24.202906855Z 2024-06-11 19:08:24,202 - __main__ - INFO - ⏰⏰ total elapsed time: 3.30 seconds +2024-06-11T19:08:24.202908439Z 2024-06-11 19:08:24,202 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.783.783.1718132806243058309.225', data=b'\n2https://help.collaboard.app/de/support-kontakieren(\x81\x80\xf4\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:24.204775189Z 2024-06-11 19:08:24,203 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.785.785.1718132806245575517.223', data=b'\n+https://help.collaboard.app/change-language(\x81\x80\xf8\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:24.204788564Z 2024-06-11 19:08:24,203 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:24.204790189Z 2024-06-11 19:08:24,203 - __main__ - INFO - message: +2024-06-11T19:08:24.204791314Z url: "https://help.collaboard.app/change-language" +2024-06-11T19:08:24.204792355Z document_id: 976690891544461313 +2024-06-11T19:08:24.204793230Z connector_id: 976341399530373121 +2024-06-11T19:08:24.204794189Z file_type: URL +2024-06-11T19:08:24.204795105Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:24.204796147Z +2024-06-11T19:08:24.204797022Z 2024-06-11 19:08:24,203 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:24.204798022Z 2024-06-11 19:08:24,203 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:24.238118897Z 2024-06-11 19:08:24,237 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:24.238137147Z 2024-06-11 19:08:24,237 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/change-language +2024-06-11T19:08:24.238138980Z 2024-06-11 19:08:24,238 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/change-language +2024-06-11T19:08:24.439858314Z 2024-06-11 19:08:24,439 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.20 seconds +2024-06-11T19:08:24.458565064Z 2024-06-11 19:08:24,458 - Milvus_DB - INFO - deleting all entities related to document 976690891544461313 +2024-06-11T19:08:24.473662064Z 2024-06-11 19:08:24,473 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:08:24.485981605Z 2024-06-11 19:08:24,485 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:24.485997522Z 2024-06-11 19:08:24,485 - root - INFO - splitting the entity +2024-06-11T19:08:24.485999022Z 2024-06-11 19:08:24,485 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:24.486291439Z 2024-06-11 19:08:24,486 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/change-language +2024-06-11T19:08:24.486300272Z 2024-06-11 19:08:24,486 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:08:25.092163189Z 2024-06-11 19:08:25,092 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:08:28.193105816Z 2024-06-11 19:08:28,192 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:08:28.199682816Z 2024-06-11 19:08:28,199 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132906.932157940,0): "sql txn" meta={id=f78ae857 key=/Table/121/1/976691226850033665/0 iso=Serializable pri=0.00149934 epo=0 ts=1718132907.097115273,1 min=1718132904.491892355,0 seq=1} lock=true stat=PENDING rts=1718132904.491892355,0 wto=false gul=1718132904.991892355,0 +2024-06-11T19:08:28.199858399Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:28.199865024Z +2024-06-11T19:08:28.199866191Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:28.199867316Z 2024-06-11 19:08:28,199 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:28.206632149Z 2024-06-11 19:08:28,206 - __main__ - INFO - 1 +2024-06-11T19:08:28.207770441Z 2024-06-11 19:08:28,207 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:28.207780274Z 2024-06-11 19:08:28,207 - __main__ - INFO - ⏰⏰ total elapsed time: 4.00 seconds +2024-06-11T19:08:28.207782607Z 2024-06-11 19:08:28,207 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.785.785.1718132806245575517.223', data=b'\n+https://help.collaboard.app/change-language(\x81\x80\xf8\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:28.208548732Z 2024-06-11 19:08:28,208 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.787.787.1718132806247554809.221', data=b'\n*https://help.collaboard.app/login-problems(\x81\x80\xfc\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:28.208552982Z 2024-06-11 19:08:28,208 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:28.208554191Z 2024-06-11 19:08:28,208 - __main__ - INFO - message: +2024-06-11T19:08:28.208555149Z url: "https://help.collaboard.app/login-problems" +2024-06-11T19:08:28.208556149Z document_id: 976690891544526849 +2024-06-11T19:08:28.208557066Z connector_id: 976341399530373121 +2024-06-11T19:08:28.208557941Z file_type: URL +2024-06-11T19:08:28.208558816Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:28.208559857Z +2024-06-11T19:08:28.208560732Z 2024-06-11 19:08:28,208 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:28.208561732Z 2024-06-11 19:08:28,208 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:28.245305649Z 2024-06-11 19:08:28,245 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:28.245345066Z 2024-06-11 19:08:28,245 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/login-problems +2024-06-11T19:08:28.245355566Z 2024-06-11 19:08:28,245 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/login-problems +2024-06-11T19:08:28.357996649Z 2024-06-11 19:08:28,357 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:08:28.369328316Z 2024-06-11 19:08:28,369 - Milvus_DB - INFO - deleting all entities related to document 976690891544526849 +2024-06-11T19:08:28.381402024Z 2024-06-11 19:08:28,381 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:28.391650774Z 2024-06-11 19:08:28,391 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:28.391657232Z 2024-06-11 19:08:28,391 - root - INFO - splitting the entity +2024-06-11T19:08:28.391658649Z 2024-06-11 19:08:28,391 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:28.392150107Z 2024-06-11 19:08:28,391 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/login-problems +2024-06-11T19:08:28.392154149Z 2024-06-11 19:08:28,391 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:08:28.974074899Z 2024-06-11 19:08:28,973 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:08:32.078652776Z 2024-06-11 19:08:32,078 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:08:32.086158942Z 2024-06-11 19:08:32,085 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132910.953797275,0): "sql txn" meta={id=3db89ca3 key=/Table/121/1/976691239584169985/0 iso=Serializable pri=0.01431904 epo=0 ts=1718132911.198548192,1 min=1718132908.396746941,0 seq=1} lock=true stat=PENDING rts=1718132908.396746941,0 wto=false gul=1718132908.896746941,0 +2024-06-11T19:08:32.086169067Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:32.086170776Z +2024-06-11T19:08:32.086171692Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:32.086172734Z 2024-06-11 19:08:32,086 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:32.092029234Z 2024-06-11 19:08:32,091 - __main__ - INFO - 1 +2024-06-11T19:08:32.094538567Z 2024-06-11 19:08:32,094 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:32.094552567Z 2024-06-11 19:08:32,094 - __main__ - INFO - ⏰⏰ total elapsed time: 3.89 seconds +2024-06-11T19:08:32.094554151Z 2024-06-11 19:08:32,094 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.787.787.1718132806247554809.221', data=b'\n*https://help.collaboard.app/login-problems(\x81\x80\xfc\xee\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:32.095401651Z 2024-06-11 19:08:32,094 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.789.789.1718132806249971225.219', data=b'\nDhttps://help.collaboard.app/de/login-probleme-unautorisierter-fehler(\x81\x80\x80\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:32.095414901Z 2024-06-11 19:08:32,094 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:32.095428067Z 2024-06-11 19:08:32,094 - __main__ - INFO - message: +2024-06-11T19:08:32.095429484Z url: "https://help.collaboard.app/de/login-probleme-unautorisierter-fehler" +2024-06-11T19:08:32.095430734Z document_id: 976690891544592385 +2024-06-11T19:08:32.095431692Z connector_id: 976341399530373121 +2024-06-11T19:08:32.095432609Z file_type: URL +2024-06-11T19:08:32.095433526Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:32.095434567Z +2024-06-11T19:08:32.095435442Z 2024-06-11 19:08:32,094 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:32.095436526Z 2024-06-11 19:08:32,094 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:32.137415901Z 2024-06-11 19:08:32,136 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:32.137434109Z 2024-06-11 19:08:32,137 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/login-probleme-unautorisierter-fehler +2024-06-11T19:08:32.137435651Z 2024-06-11 19:08:32,137 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/login-probleme-unautorisierter-fehler +2024-06-11T19:08:32.237426067Z 2024-06-11 19:08:32,237 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:08:32.249864984Z 2024-06-11 19:08:32,249 - Milvus_DB - INFO - deleting all entities related to document 976690891544592385 +2024-06-11T19:08:32.261666484Z 2024-06-11 19:08:32,261 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:32.270619067Z 2024-06-11 19:08:32,270 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:32.270626067Z 2024-06-11 19:08:32,270 - root - INFO - splitting the entity +2024-06-11T19:08:32.270627401Z 2024-06-11 19:08:32,270 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:32.270927276Z 2024-06-11 19:08:32,270 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/login-probleme-unautorisierter-fehler +2024-06-11T19:08:32.270930234Z 2024-06-11 19:08:32,270 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:08:32.826159901Z 2024-06-11 19:08:32,826 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:08:35.909295153Z 2024-06-11 19:08:35,909 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:08:35.918769986Z 2024-06-11 19:08:35,918 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132914.990611235,0): "sql txn" meta={id=88018595 key=/Table/121/1/976691252134412289/0 iso=Serializable pri=0.01731412 epo=0 ts=1718132915.160614944,1 min=1718132912.276879359,0 seq=1} lock=true stat=PENDING rts=1718132912.276879359,0 wto=false gul=1718132912.776879359,0 +2024-06-11T19:08:35.918779611Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:35.918780861Z +2024-06-11T19:08:35.918781944Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:35.918782944Z 2024-06-11 19:08:35,918 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:35.960842903Z 2024-06-11 19:08:35,960 - __main__ - INFO - 1 +2024-06-11T19:08:35.962835236Z 2024-06-11 19:08:35,961 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:35.962840653Z 2024-06-11 19:08:35,961 - __main__ - INFO - ⏰⏰ total elapsed time: 3.87 seconds +2024-06-11T19:08:35.962841944Z 2024-06-11 19:08:35,961 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.789.789.1718132806249971225.219', data=b'\nDhttps://help.collaboard.app/de/login-probleme-unautorisierter-fehler(\x81\x80\x80\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:35.962844278Z 2024-06-11 19:08:35,962 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.791.791.1718132806253038725.217', data=b'\n9https://help.collaboard.app/do-you-have-connection-issues(\x81\x80\x84\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:35.962845986Z 2024-06-11 19:08:35,962 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:35.962846944Z 2024-06-11 19:08:35,962 - __main__ - INFO - message: +2024-06-11T19:08:35.962848194Z url: "https://help.collaboard.app/do-you-have-connection-issues" +2024-06-11T19:08:35.962849236Z document_id: 976690891544657921 +2024-06-11T19:08:35.962850153Z connector_id: 976341399530373121 +2024-06-11T19:08:35.962851028Z file_type: URL +2024-06-11T19:08:35.962851903Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:35.962852944Z +2024-06-11T19:08:35.962853778Z 2024-06-11 19:08:35,962 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:35.962854778Z 2024-06-11 19:08:35,962 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:35.998586403Z 2024-06-11 19:08:35,998 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:35.998601653Z 2024-06-11 19:08:35,998 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/do-you-have-connection-issues +2024-06-11T19:08:35.998603194Z 2024-06-11 19:08:35,998 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/do-you-have-connection-issues +2024-06-11T19:08:36.097735778Z 2024-06-11 19:08:36,097 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:08:36.110135403Z 2024-06-11 19:08:36,109 - Milvus_DB - INFO - deleting all entities related to document 976690891544657921 +2024-06-11T19:08:36.122080903Z 2024-06-11 19:08:36,121 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:36.131927986Z 2024-06-11 19:08:36,131 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:36.131936069Z 2024-06-11 19:08:36,131 - root - INFO - splitting the entity +2024-06-11T19:08:36.131938444Z 2024-06-11 19:08:36,131 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:36.132200694Z 2024-06-11 19:08:36,132 - URLSemantic - INFO - created 13 chunks for https://help.collaboard.app/do-you-have-connection-issues +2024-06-11T19:08:36.132204486Z 2024-06-11 19:08:36,132 - root - INFO - saving in milvus 13 chunk(s) +2024-06-11T19:08:36.693390486Z 2024-06-11 19:08:36,693 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:08:39.745992904Z 2024-06-11 19:08:39,745 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:08:39.754518529Z 2024-06-11 19:08:39,754 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132918.953247487,0): "sql txn" meta={id=c0ac0950 key=/Table/121/1/976691264706936833/0 iso=Serializable pri=0.06291196 epo=0 ts=1718132919.099293696,1 min=1718132916.136827403,0 seq=1} lock=true stat=PENDING rts=1718132916.136827403,0 wto=false gul=1718132916.636827403,0 +2024-06-11T19:08:39.754533363Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:39.754534988Z +2024-06-11T19:08:39.754536154Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:39.754537404Z 2024-06-11 19:08:39,754 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:39.763452279Z 2024-06-11 19:08:39,763 - __main__ - INFO - 1 +2024-06-11T19:08:39.764321196Z 2024-06-11 19:08:39,763 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:39.764326988Z 2024-06-11 19:08:39,764 - __main__ - INFO - ⏰⏰ total elapsed time: 3.80 seconds +2024-06-11T19:08:39.764328779Z 2024-06-11 19:08:39,764 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.791.791.1718132806253038725.217', data=b'\n9https://help.collaboard.app/do-you-have-connection-issues(\x81\x80\x84\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:39.766367071Z 2024-06-11 19:08:39,765 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.793.793.1718132806255533059.215', data=b'\n7https://help.collaboard.app/de/unterst%C3%BCtze-browser(\x81\x80\x88\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:39.766447946Z 2024-06-11 19:08:39,766 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:39.766886029Z 2024-06-11 19:08:39,766 - __main__ - INFO - message: +2024-06-11T19:08:39.766887988Z url: "https://help.collaboard.app/de/unterst%C3%BCtze-browser" +2024-06-11T19:08:39.766889113Z document_id: 976690891544723457 +2024-06-11T19:08:39.766890113Z connector_id: 976341399530373121 +2024-06-11T19:08:39.766891071Z file_type: URL +2024-06-11T19:08:39.766891988Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:39.766893029Z +2024-06-11T19:08:39.766893988Z 2024-06-11 19:08:39,766 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:39.766895071Z 2024-06-11 19:08:39,766 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:39.808018988Z 2024-06-11 19:08:39,807 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:39.808031446Z 2024-06-11 19:08:39,808 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/unterst%C3%BCtze-browser +2024-06-11T19:08:39.808096571Z 2024-06-11 19:08:39,808 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/unterst%C3%BCtze-browser +2024-06-11T19:08:40.069209655Z 2024-06-11 19:08:40,068 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.26 seconds +2024-06-11T19:08:40.085757196Z 2024-06-11 19:08:40,085 - Milvus_DB - INFO - deleting all entities related to document 976690891544723457 +2024-06-11T19:08:40.099385988Z 2024-06-11 19:08:40,099 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:40.111447821Z 2024-06-11 19:08:40,111 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:40.111455655Z 2024-06-11 19:08:40,111 - root - INFO - splitting the entity +2024-06-11T19:08:40.111457363Z 2024-06-11 19:08:40,111 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:40.111797030Z 2024-06-11 19:08:40,111 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/de/unterst%C3%BCtze-browser +2024-06-11T19:08:40.111799905Z 2024-06-11 19:08:40,111 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:08:40.749401821Z 2024-06-11 19:08:40,748 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:08:41.322018905Z 127.0.0.1 - - [11/Jun/2024 19:08:41] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:08:41.322089572Z 2024-06-11 19:08:41,321 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:08:43.774594365Z 2024-06-11 19:08:43,773 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:08:43.806427573Z 2024-06-11 19:08:43,806 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132922.911656823,0): "sql txn" meta={id=20068f51 key=/Table/121/1/976691277906378753/0 iso=Serializable pri=0.00813371 epo=0 ts=1718132923.138523656,1 min=1718132920.117400113,0 seq=1} lock=true stat=PENDING rts=1718132920.117400113,0 wto=false gul=1718132920.617400113,0 +2024-06-11T19:08:43.806438698Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:43.806440115Z +2024-06-11T19:08:43.806441156Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:43.806671781Z 2024-06-11 19:08:43,806 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:43.825631781Z 2024-06-11 19:08:43,825 - __main__ - INFO - 1 +2024-06-11T19:08:43.826626281Z 2024-06-11 19:08:43,826 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:43.826633531Z 2024-06-11 19:08:43,826 - __main__ - INFO - ⏰⏰ total elapsed time: 4.06 seconds +2024-06-11T19:08:43.826635198Z 2024-06-11 19:08:43,826 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.793.793.1718132806255533059.215', data=b'\n7https://help.collaboard.app/de/unterst%C3%BCtze-browser(\x81\x80\x88\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:43.827372156Z 2024-06-11 19:08:43,827 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.795.795.1718132806258578809.213', data=b'\n3https://help.collaboard.app/de/unterst\xc3\xbctze-browser(\x81\x80\x8c\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:43.828150031Z 2024-06-11 19:08:43,827 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:43.828153406Z 2024-06-11 19:08:43,827 - __main__ - INFO - message: +2024-06-11T19:08:43.828154531Z url: "https://help.collaboard.app/de/unterstütze-browser" +2024-06-11T19:08:43.828155990Z document_id: 976690891544788993 +2024-06-11T19:08:43.828157156Z connector_id: 976341399530373121 +2024-06-11T19:08:43.828158115Z file_type: URL +2024-06-11T19:08:43.828158990Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:43.828160073Z +2024-06-11T19:08:43.828160948Z 2024-06-11 19:08:43,827 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:43.828161948Z 2024-06-11 19:08:43,827 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:43.871143531Z 2024-06-11 19:08:43,870 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:43.871156198Z 2024-06-11 19:08:43,871 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/unterstütze-browser +2024-06-11T19:08:43.871163240Z 2024-06-11 19:08:43,871 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/unterstütze-browser +2024-06-11T19:08:43.992945573Z 2024-06-11 19:08:43,992 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:08:44.006914365Z 2024-06-11 19:08:44,006 - Milvus_DB - INFO - deleting all entities related to document 976690891544788993 +2024-06-11T19:08:44.019597573Z 2024-06-11 19:08:44,019 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:44.033664656Z 2024-06-11 19:08:44,033 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:44.033671531Z 2024-06-11 19:08:44,033 - root - INFO - splitting the entity +2024-06-11T19:08:44.033672698Z 2024-06-11 19:08:44,033 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:44.033760948Z 2024-06-11 19:08:44,033 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/de/unterstütze-browser +2024-06-11T19:08:44.033764615Z 2024-06-11 19:08:44,033 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:08:44.601628823Z 2024-06-11 19:08:44,601 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:08:48.123982492Z 2024-06-11 19:08:48,123 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.08 seconds +2024-06-11T19:08:48.130887742Z 2024-06-11 19:08:48,130 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132926.933544449,0): "sql txn" meta={id=93021359 key=/Table/121/1/976691292159475713/0 iso=Serializable pri=0.05836515 epo=0 ts=1718132927.088637908,1 min=1718132924.038970448,0 seq=1} lock=true stat=PENDING rts=1718132924.038970448,0 wto=false gul=1718132924.538970448,0 +2024-06-11T19:08:48.130895367Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:48.130896742Z +2024-06-11T19:08:48.130897700Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:48.130898867Z 2024-06-11 19:08:48,130 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:48.138015575Z 2024-06-11 19:08:48,137 - __main__ - INFO - 1 +2024-06-11T19:08:48.139406325Z 2024-06-11 19:08:48,138 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:48.139411242Z 2024-06-11 19:08:48,138 - __main__ - INFO - ⏰⏰ total elapsed time: 4.31 seconds +2024-06-11T19:08:48.139413367Z 2024-06-11 19:08:48,138 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.795.795.1718132806258578809.213', data=b'\n3https://help.collaboard.app/de/unterst\xc3\xbctze-browser(\x81\x80\x8c\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:48.140408450Z 2024-06-11 19:08:48,139 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.797.797.1718132806261618892.211', data=b'\n0https://help.collaboard.app/de/benutzer-l\xc3\xb6schen(\x81\x80\x90\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:48.140413575Z 2024-06-11 19:08:48,139 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:48.140414992Z 2024-06-11 19:08:48,139 - __main__ - INFO - message: +2024-06-11T19:08:48.140416033Z url: "https://help.collaboard.app/de/benutzer-löschen" +2024-06-11T19:08:48.140417158Z document_id: 976690891544854529 +2024-06-11T19:08:48.140418075Z connector_id: 976341399530373121 +2024-06-11T19:08:48.140418992Z file_type: URL +2024-06-11T19:08:48.140419867Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:48.140420908Z +2024-06-11T19:08:48.140421783Z 2024-06-11 19:08:48,139 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:48.140422825Z 2024-06-11 19:08:48,139 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:48.224706117Z 2024-06-11 19:08:48,223 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:48.224726575Z 2024-06-11 19:08:48,223 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/benutzer-löschen +2024-06-11T19:08:48.224728700Z 2024-06-11 19:08:48,223 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/benutzer-löschen +2024-06-11T19:08:48.321721575Z 2024-06-11 19:08:48,321 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:08:48.334706783Z 2024-06-11 19:08:48,334 - Milvus_DB - INFO - deleting all entities related to document 976690891544854529 +2024-06-11T19:08:48.347377283Z 2024-06-11 19:08:48,347 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:48.360178992Z 2024-06-11 19:08:48,360 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:48.360201200Z 2024-06-11 19:08:48,360 - root - INFO - splitting the entity +2024-06-11T19:08:48.360203783Z 2024-06-11 19:08:48,360 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:48.360424700Z 2024-06-11 19:08:48,360 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/benutzer-löschen +2024-06-11T19:08:48.360427700Z 2024-06-11 19:08:48,360 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:08:48.927852325Z 2024-06-11 19:08:48,927 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:08:51.968252202Z 2024-06-11 19:08:51,967 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.60 seconds +2024-06-11T19:08:51.975894827Z 2024-06-11 19:08:51,975 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132930.926781160,0): "sql txn" meta={id=681dd719 key=/Table/121/1/976691304757395457/0 iso=Serializable pri=0.08067801 epo=0 ts=1718132931.076893160,1 min=1718132928.368286742,0 seq=1} lock=true stat=PENDING rts=1718132928.368286742,0 wto=false gul=1718132928.868286742,0 +2024-06-11T19:08:51.975903494Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:51.975905077Z +2024-06-11T19:08:51.975906160Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:51.975907202Z 2024-06-11 19:08:51,975 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:51.984907827Z 2024-06-11 19:08:51,984 - __main__ - INFO - 1 +2024-06-11T19:08:51.986272744Z 2024-06-11 19:08:51,985 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:51.986277785Z 2024-06-11 19:08:51,985 - __main__ - INFO - ⏰⏰ total elapsed time: 3.85 seconds +2024-06-11T19:08:51.986279327Z 2024-06-11 19:08:51,985 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.797.797.1718132806261618892.211', data=b'\n0https://help.collaboard.app/de/benutzer-l\xc3\xb6schen(\x81\x80\x90\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:51.986657660Z 2024-06-11 19:08:51,986 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.799.799.1718132806264883975.209', data=b'\nDhttps://help.collaboard.app/de/authentifizierungsmodus-in-collaboard(\x81\x80\x94\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:51.986662244Z 2024-06-11 19:08:51,986 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:51.986663619Z 2024-06-11 19:08:51,986 - __main__ - INFO - message: +2024-06-11T19:08:51.986664785Z url: "https://help.collaboard.app/de/authentifizierungsmodus-in-collaboard" +2024-06-11T19:08:51.986666035Z document_id: 976690891544920065 +2024-06-11T19:08:51.986666994Z connector_id: 976341399530373121 +2024-06-11T19:08:51.986669869Z file_type: URL +2024-06-11T19:08:51.986670785Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:51.986671910Z +2024-06-11T19:08:51.986672785Z 2024-06-11 19:08:51,986 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:51.986673827Z 2024-06-11 19:08:51,986 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:52.034488035Z 2024-06-11 19:08:52,033 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:52.034500119Z 2024-06-11 19:08:52,033 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/authentifizierungsmodus-in-collaboard +2024-06-11T19:08:52.034501827Z 2024-06-11 19:08:52,033 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/authentifizierungsmodus-in-collaboard +2024-06-11T19:08:52.283728952Z 2024-06-11 19:08:52,283 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.25 seconds +2024-06-11T19:08:52.294750410Z 2024-06-11 19:08:52,294 - Milvus_DB - INFO - deleting all entities related to document 976690891544920065 +2024-06-11T19:08:52.306339660Z 2024-06-11 19:08:52,306 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:52.315767910Z 2024-06-11 19:08:52,315 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:52.315774494Z 2024-06-11 19:08:52,315 - root - INFO - splitting the entity +2024-06-11T19:08:52.315775910Z 2024-06-11 19:08:52,315 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:52.315777244Z 2024-06-11 19:08:52,315 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/authentifizierungsmodus-in-collaboard +2024-06-11T19:08:52.315778369Z 2024-06-11 19:08:52,315 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:08:52.870195119Z 2024-06-11 19:08:52,870 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:08:55.963176162Z 2024-06-11 19:08:55,962 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:08:55.967749620Z 2024-06-11 19:08:55,967 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132934.862534037,0): "sql txn" meta={id=8aaba1f3 key=/Table/121/1/976691317847293953/0 iso=Serializable pri=0.02820799 epo=0 ts=1718132935.095216870,1 min=1718132932.319622744,0 seq=1} lock=true stat=PENDING rts=1718132932.319622744,0 wto=false gul=1718132932.819622744,0 +2024-06-11T19:08:55.967761370Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:08:55.967762912Z +2024-06-11T19:08:55.967764120Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:08:55.967765245Z 2024-06-11 19:08:55,967 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:08:55.973006329Z 2024-06-11 19:08:55,972 - __main__ - INFO - 1 +2024-06-11T19:08:55.974539829Z 2024-06-11 19:08:55,973 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:08:55.974545454Z 2024-06-11 19:08:55,973 - __main__ - INFO - ⏰⏰ total elapsed time: 3.99 seconds +2024-06-11T19:08:55.974546787Z 2024-06-11 19:08:55,974 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.799.799.1718132806264883975.209', data=b'\nDhttps://help.collaboard.app/de/authentifizierungsmodus-in-collaboard(\x81\x80\x94\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:08:55.975740662Z 2024-06-11 19:08:55,975 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.801.801.1718132806267383434.207', data=b'\n5https://help.collaboard.app/de/passwort-zur\xc3\xbccksetzen(\x81\x80\x98\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:08:55.975746245Z 2024-06-11 19:08:55,975 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:08:55.975747495Z 2024-06-11 19:08:55,975 - __main__ - INFO - message: +2024-06-11T19:08:55.975750912Z url: "https://help.collaboard.app/de/passwort-zurücksetzen" +2024-06-11T19:08:55.975752162Z document_id: 976690891544985601 +2024-06-11T19:08:55.975753120Z connector_id: 976341399530373121 +2024-06-11T19:08:55.975754037Z file_type: URL +2024-06-11T19:08:55.975755037Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:08:55.975756079Z +2024-06-11T19:08:55.975756954Z 2024-06-11 19:08:55,975 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:08:55.975757954Z 2024-06-11 19:08:55,975 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:08:56.010291662Z 2024-06-11 19:08:56,009 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:08:56.010665204Z 2024-06-11 19:08:56,010 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/passwort-zurücksetzen +2024-06-11T19:08:56.010669620Z 2024-06-11 19:08:56,010 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/passwort-zurücksetzen +2024-06-11T19:08:56.088832537Z 2024-06-11 19:08:56,088 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.08 seconds +2024-06-11T19:08:56.099043412Z 2024-06-11 19:08:56,098 - Milvus_DB - INFO - deleting all entities related to document 976690891544985601 +2024-06-11T19:08:56.112026370Z 2024-06-11 19:08:56,111 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:08:56.120815329Z 2024-06-11 19:08:56,120 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:08:56.120821995Z 2024-06-11 19:08:56,120 - root - INFO - splitting the entity +2024-06-11T19:08:56.120823412Z 2024-06-11 19:08:56,120 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:08:56.121124537Z 2024-06-11 19:08:56,120 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/passwort-zurücksetzen +2024-06-11T19:08:56.121127579Z 2024-06-11 19:08:56,120 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:08:56.681985204Z 2024-06-11 19:08:56,681 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:09:00.325476664Z 2024-06-11 19:09:00,324 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.20 seconds +2024-06-11T19:09:00.343665456Z 2024-06-11 19:09:00,343 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132938.936316622,0): "sql txn" meta={id=99df015c key=/Table/121/1/976691332156489729/0 iso=Serializable pri=0.00059870 epo=0 ts=1718132939.088917205,1 min=1718132936.125628745,0 seq=1} lock=true stat=PENDING rts=1718132936.125628745,0 wto=false gul=1718132936.625628745,0 +2024-06-11T19:09:00.343687414Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:00.343690831Z +2024-06-11T19:09:00.343693039Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:00.343695331Z 2024-06-11 19:09:00,343 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:00.354018831Z 2024-06-11 19:09:00,353 - __main__ - INFO - 1 +2024-06-11T19:09:00.357183789Z 2024-06-11 19:09:00,355 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:00.357200873Z 2024-06-11 19:09:00,355 - __main__ - INFO - ⏰⏰ total elapsed time: 4.38 seconds +2024-06-11T19:09:00.357205039Z 2024-06-11 19:09:00,355 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.801.801.1718132806267383434.207', data=b'\n5https://help.collaboard.app/de/passwort-zur\xc3\xbccksetzen(\x81\x80\x98\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:00.358057831Z 2024-06-11 19:09:00,357 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.803.803.1718132806269587934.205', data=b'\n7https://help.collaboard.app/de/benutzer-zusammenf\xc3\xbchren(\x81\x80\x9c\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:00.358071456Z 2024-06-11 19:09:00,357 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:00.358074581Z 2024-06-11 19:09:00,357 - __main__ - INFO - message: +2024-06-11T19:09:00.358076914Z url: "https://help.collaboard.app/de/benutzer-zusammenführen" +2024-06-11T19:09:00.358079123Z document_id: 976690891545051137 +2024-06-11T19:09:00.358080873Z connector_id: 976341399530373121 +2024-06-11T19:09:00.358082623Z file_type: URL +2024-06-11T19:09:00.358084331Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:00.358086998Z +2024-06-11T19:09:00.358088748Z 2024-06-11 19:09:00,357 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:00.358090831Z 2024-06-11 19:09:00,357 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:00.401575498Z 2024-06-11 19:09:00,401 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:00.401590789Z 2024-06-11 19:09:00,401 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/benutzer-zusammenführen +2024-06-11T19:09:00.401592789Z 2024-06-11 19:09:00,401 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/benutzer-zusammenführen +2024-06-11T19:09:00.551223164Z 2024-06-11 19:09:00,551 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.15 seconds +2024-06-11T19:09:00.564475498Z 2024-06-11 19:09:00,564 - Milvus_DB - INFO - deleting all entities related to document 976690891545051137 +2024-06-11T19:09:00.576244373Z 2024-06-11 19:09:00,576 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:00.587179873Z 2024-06-11 19:09:00,586 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:00.587187456Z 2024-06-11 19:09:00,587 - root - INFO - splitting the entity +2024-06-11T19:09:00.587189206Z 2024-06-11 19:09:00,587 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:00.587539914Z 2024-06-11 19:09:00,587 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/benutzer-zusammenführen +2024-06-11T19:09:00.587544248Z 2024-06-11 19:09:00,587 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:09:01.154914331Z 2024-06-11 19:09:01,154 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:09:04.210286166Z 2024-06-11 19:09:04,210 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:09:04.215871499Z 2024-06-11 19:09:04,215 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132942.933854207,0): "sql txn" meta={id=7842dcc9 key=/Table/121/1/976691344871915521/0 iso=Serializable pri=0.02821670 epo=0 ts=1718132943.108077790,1 min=1718132940.591736081,0 seq=1} lock=true stat=PENDING rts=1718132940.591736081,0 wto=false gul=1718132941.091736081,0 +2024-06-11T19:09:04.215884249Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:04.215886249Z +2024-06-11T19:09:04.215887374Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:04.215888458Z 2024-06-11 19:09:04,215 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:04.220304583Z 2024-06-11 19:09:04,220 - __main__ - INFO - 1 +2024-06-11T19:09:04.221365249Z 2024-06-11 19:09:04,220 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:04.221373458Z 2024-06-11 19:09:04,221 - __main__ - INFO - ⏰⏰ total elapsed time: 3.86 seconds +2024-06-11T19:09:04.221374958Z 2024-06-11 19:09:04,221 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.803.803.1718132806269587934.205', data=b'\n7https://help.collaboard.app/de/benutzer-zusammenf\xc3\xbchren(\x81\x80\x9c\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:04.222357999Z 2024-06-11 19:09:04,221 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.805.805.1718132806271626017.203', data=b'\nJhttps://help.collaboard.app/de/wie-erstelle-ich-ein-neues-collaboard-konto(\x81\x80\xa0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:04.222364541Z 2024-06-11 19:09:04,221 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:04.222365916Z 2024-06-11 19:09:04,221 - __main__ - INFO - message: +2024-06-11T19:09:04.222366958Z url: "https://help.collaboard.app/de/wie-erstelle-ich-ein-neues-collaboard-konto" +2024-06-11T19:09:04.222368124Z document_id: 976690891545116673 +2024-06-11T19:09:04.222369124Z connector_id: 976341399530373121 +2024-06-11T19:09:04.222372749Z file_type: URL +2024-06-11T19:09:04.222373791Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:04.222374791Z +2024-06-11T19:09:04.222375749Z 2024-06-11 19:09:04,222 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:04.222376833Z 2024-06-11 19:09:04,222 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:04.260934791Z 2024-06-11 19:09:04,260 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:04.260956374Z 2024-06-11 19:09:04,260 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-erstelle-ich-ein-neues-collaboard-konto +2024-06-11T19:09:04.260957958Z 2024-06-11 19:09:04,260 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-erstelle-ich-ein-neues-collaboard-konto +2024-06-11T19:09:04.371502208Z 2024-06-11 19:09:04,371 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:09:04.386255333Z 2024-06-11 19:09:04,386 - Milvus_DB - INFO - deleting all entities related to document 976690891545116673 +2024-06-11T19:09:04.397684374Z 2024-06-11 19:09:04,397 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:04.406733749Z 2024-06-11 19:09:04,406 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:04.406739958Z 2024-06-11 19:09:04,406 - root - INFO - splitting the entity +2024-06-11T19:09:04.406741291Z 2024-06-11 19:09:04,406 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:04.406896874Z 2024-06-11 19:09:04,406 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/wie-erstelle-ich-ein-neues-collaboard-konto +2024-06-11T19:09:04.406902124Z 2024-06-11 19:09:04,406 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:09:04.995465166Z 2024-06-11 19:09:04,995 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:09:08.075956210Z 2024-06-11 19:09:08,075 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:09:08.082233751Z 2024-06-11 19:09:08,081 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132946.881526667,0): "sql txn" meta={id=6ce23944 key=/Table/121/1/976691357538385921/0 iso=Serializable pri=0.03781966 epo=0 ts=1718132947.103950126,1 min=1718132944.410777333,0 seq=1} lock=true stat=PENDING rts=1718132944.410777333,0 wto=false gul=1718132944.910777333,0 +2024-06-11T19:09:08.082241918Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:08.082243751Z +2024-06-11T19:09:08.082245126Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:08.082246585Z 2024-06-11 19:09:08,082 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:08.090045293Z 2024-06-11 19:09:08,089 - __main__ - INFO - 1 +2024-06-11T19:09:08.091979960Z 2024-06-11 19:09:08,090 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:08.091996876Z 2024-06-11 19:09:08,090 - __main__ - INFO - ⏰⏰ total elapsed time: 3.87 seconds +2024-06-11T19:09:08.091998960Z 2024-06-11 19:09:08,090 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.805.805.1718132806271626017.203', data=b'\nJhttps://help.collaboard.app/de/wie-erstelle-ich-ein-neues-collaboard-konto(\x81\x80\xa0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:08.092075210Z 2024-06-11 19:09:08,091 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.807.807.1718132806274398642.201', data=b'\nLhttps://help.collaboard.app/de/wie-kann-der-projektbesitzer-ge\xc3\xa4ndert-werden(\x81\x80\xa4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:08.092080626Z 2024-06-11 19:09:08,091 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:08.092082043Z 2024-06-11 19:09:08,091 - __main__ - INFO - message: +2024-06-11T19:09:08.092083293Z url: "https://help.collaboard.app/de/wie-kann-der-projektbesitzer-geändert-werden" +2024-06-11T19:09:08.092084585Z document_id: 976690891545182209 +2024-06-11T19:09:08.092085710Z connector_id: 976341399530373121 +2024-06-11T19:09:08.092086876Z file_type: URL +2024-06-11T19:09:08.092088126Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:08.092089376Z +2024-06-11T19:09:08.092090460Z 2024-06-11 19:09:08,091 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:08.092099168Z 2024-06-11 19:09:08,091 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:08.140957376Z 2024-06-11 19:09:08,140 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:08.140988251Z 2024-06-11 19:09:08,140 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-kann-der-projektbesitzer-geändert-werden +2024-06-11T19:09:08.140990751Z 2024-06-11 19:09:08,140 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-kann-der-projektbesitzer-geändert-werden +2024-06-11T19:09:08.281855418Z 2024-06-11 19:09:08,281 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.14 seconds +2024-06-11T19:09:08.308602251Z 2024-06-11 19:09:08,308 - Milvus_DB - INFO - deleting all entities related to document 976690891545182209 +2024-06-11T19:09:08.322288626Z 2024-06-11 19:09:08,322 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:08.333947293Z 2024-06-11 19:09:08,333 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:08.333979168Z 2024-06-11 19:09:08,333 - root - INFO - splitting the entity +2024-06-11T19:09:08.333982751Z 2024-06-11 19:09:08,333 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:08.334658710Z 2024-06-11 19:09:08,334 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/wie-kann-der-projektbesitzer-geändert-werden +2024-06-11T19:09:08.334664501Z 2024-06-11 19:09:08,334 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:09:08.922271002Z 2024-06-11 19:09:08,922 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:09:11.982037170Z 2024-06-11 19:09:11,981 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:09:11.995477545Z 2024-06-11 19:09:11,995 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132950.910562044,0): "sql txn" meta={id=4e303ed6 key=/Table/121/1/976691370338746369/0 iso=Serializable pri=0.00709980 epo=0 ts=1718132951.297972294,1 min=1718132948.339190835,0 seq=1} lock=true stat=PENDING rts=1718132948.339190835,0 wto=false gul=1718132948.839190835,0 +2024-06-11T19:09:11.995487586Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:11.995489420Z +2024-06-11T19:09:11.995490711Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:11.996313378Z 2024-06-11 19:09:11,995 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:12.017363086Z 2024-06-11 19:09:12,017 - __main__ - INFO - 1 +2024-06-11T19:09:12.018431545Z 2024-06-11 19:09:12,018 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:12.019166461Z 2024-06-11 19:09:12,018 - __main__ - INFO - ⏰⏰ total elapsed time: 3.93 seconds +2024-06-11T19:09:12.019180920Z 2024-06-11 19:09:12,018 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.807.807.1718132806274398642.201', data=b'\nLhttps://help.collaboard.app/de/wie-kann-der-projektbesitzer-ge\xc3\xa4ndert-werden(\x81\x80\xa4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:12.019471961Z 2024-06-11 19:09:12,019 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.809.809.1718132806277522475.199', data=b'\n/https://help.collaboard.app/de/projekt-l\xc3\xb6schen(\x81\x80\xa8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:12.019477170Z 2024-06-11 19:09:12,019 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:12.019478628Z 2024-06-11 19:09:12,019 - __main__ - INFO - message: +2024-06-11T19:09:12.019479670Z url: "https://help.collaboard.app/de/projekt-löschen" +2024-06-11T19:09:12.019480795Z document_id: 976690891545247745 +2024-06-11T19:09:12.019481711Z connector_id: 976341399530373121 +2024-06-11T19:09:12.019482586Z file_type: URL +2024-06-11T19:09:12.019483670Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:12.019484711Z +2024-06-11T19:09:12.019485545Z 2024-06-11 19:09:12,019 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:12.019486545Z 2024-06-11 19:09:12,019 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:12.078588461Z 2024-06-11 19:09:12,078 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:12.078608003Z 2024-06-11 19:09:12,078 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/projekt-löschen +2024-06-11T19:09:12.078610170Z 2024-06-11 19:09:12,078 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/projekt-löschen +2024-06-11T19:09:12.220249003Z 2024-06-11 19:09:12,220 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.14 seconds +2024-06-11T19:09:12.233331753Z 2024-06-11 19:09:12,233 - Milvus_DB - INFO - deleting all entities related to document 976690891545247745 +2024-06-11T19:09:12.246289170Z 2024-06-11 19:09:12,246 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:12.257507920Z 2024-06-11 19:09:12,257 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:12.257519045Z 2024-06-11 19:09:12,257 - root - INFO - splitting the entity +2024-06-11T19:09:12.257520420Z 2024-06-11 19:09:12,257 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:12.257796045Z 2024-06-11 19:09:12,257 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/projekt-löschen +2024-06-11T19:09:12.257809587Z 2024-06-11 19:09:12,257 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:09:12.809481628Z 2024-06-11 19:09:12,809 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:09:15.872176297Z 2024-06-11 19:09:15,871 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:09:15.880601838Z 2024-06-11 19:09:15,880 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132955.124497463,0): "sql txn" meta={id=a51b1d3e key=/Table/121/1/976691383085563905/0 iso=Serializable pri=0.00823280 epo=0 ts=1718132955.314702671,1 min=1718132952.262368462,0 seq=1} lock=true stat=PENDING rts=1718132952.262368462,0 wto=false gul=1718132952.762368462,0 +2024-06-11T19:09:15.880621797Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:15.880623338Z +2024-06-11T19:09:15.880624463Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:15.880625505Z 2024-06-11 19:09:15,880 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:15.889064130Z 2024-06-11 19:09:15,888 - __main__ - INFO - 1 +2024-06-11T19:09:15.889719463Z 2024-06-11 19:09:15,889 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:15.890203255Z 2024-06-11 19:09:15,889 - __main__ - INFO - ⏰⏰ total elapsed time: 3.87 seconds +2024-06-11T19:09:15.890207130Z 2024-06-11 19:09:15,890 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.809.809.1718132806277522475.199', data=b'\n/https://help.collaboard.app/de/projekt-l\xc3\xb6schen(\x81\x80\xa8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:15.890496588Z 2024-06-11 19:09:15,890 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.811.811.1718132806279737225.197', data=b'\n8https://help.collaboard.app/de/how-do-i-rename-a-project(\x81\x80\xac\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:15.890499922Z 2024-06-11 19:09:15,890 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:15.890579755Z 2024-06-11 19:09:15,890 - __main__ - INFO - message: +2024-06-11T19:09:15.890581463Z url: "https://help.collaboard.app/de/how-do-i-rename-a-project" +2024-06-11T19:09:15.890582630Z document_id: 976690891545313281 +2024-06-11T19:09:15.890583838Z connector_id: 976341399530373121 +2024-06-11T19:09:15.890584838Z file_type: URL +2024-06-11T19:09:15.890585755Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:15.890592505Z +2024-06-11T19:09:15.890593463Z 2024-06-11 19:09:15,890 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:15.890594547Z 2024-06-11 19:09:15,890 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:15.929477088Z 2024-06-11 19:09:15,929 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:15.929493838Z 2024-06-11 19:09:15,929 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/how-do-i-rename-a-project +2024-06-11T19:09:15.929495672Z 2024-06-11 19:09:15,929 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/how-do-i-rename-a-project +2024-06-11T19:09:16.087426338Z 2024-06-11 19:09:16,087 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.16 seconds +2024-06-11T19:09:16.102674088Z 2024-06-11 19:09:16,102 - Milvus_DB - INFO - deleting all entities related to document 976690891545313281 +2024-06-11T19:09:16.116750422Z 2024-06-11 19:09:16,116 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:16.128735380Z 2024-06-11 19:09:16,128 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:16.128754755Z 2024-06-11 19:09:16,128 - root - INFO - splitting the entity +2024-06-11T19:09:16.128756255Z 2024-06-11 19:09:16,128 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:16.128843297Z 2024-06-11 19:09:16,128 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/de/how-do-i-rename-a-project +2024-06-11T19:09:16.128845630Z 2024-06-11 19:09:16,128 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:09:16.695576589Z 2024-06-11 19:09:16,695 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:09:19.753155923Z 2024-06-11 19:09:19,752 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:09:19.757290007Z 2024-06-11 19:09:19,757 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132959.093257340,0): "sql txn" meta={id=1be26ef4 key=/Table/121/1/976691395802136577/0 iso=Serializable pri=0.02444070 epo=0 ts=1718132959.258739048,1 min=1718132956.132904547,0 seq=1} lock=true stat=PENDING rts=1718132956.132904547,0 wto=false gul=1718132956.632904547,0 +2024-06-11T19:09:19.757297965Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:19.757299632Z +2024-06-11T19:09:19.757300840Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:19.757302090Z 2024-06-11 19:09:19,757 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:19.761531048Z 2024-06-11 19:09:19,761 - __main__ - INFO - 1 +2024-06-11T19:09:19.763035423Z 2024-06-11 19:09:19,762 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:19.763046715Z 2024-06-11 19:09:19,762 - __main__ - INFO - ⏰⏰ total elapsed time: 3.87 seconds +2024-06-11T19:09:19.763048340Z 2024-06-11 19:09:19,762 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.811.811.1718132806279737225.197', data=b'\n8https://help.collaboard.app/de/how-do-i-rename-a-project(\x81\x80\xac\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:19.764506048Z 2024-06-11 19:09:19,762 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.813.813.1718132806282375975.195', data=b'\n/https://help.collaboard.app/de/projekt-kopieren(\x81\x80\xb0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:19.764511757Z 2024-06-11 19:09:19,762 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:19.764513132Z 2024-06-11 19:09:19,762 - __main__ - INFO - message: +2024-06-11T19:09:19.764514257Z url: "https://help.collaboard.app/de/projekt-kopieren" +2024-06-11T19:09:19.764515632Z document_id: 976690891545378817 +2024-06-11T19:09:19.764516548Z connector_id: 976341399530373121 +2024-06-11T19:09:19.764517423Z file_type: URL +2024-06-11T19:09:19.764518298Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:19.764519298Z +2024-06-11T19:09:19.764520173Z 2024-06-11 19:09:19,762 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:19.764521215Z 2024-06-11 19:09:19,762 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:19.796599507Z 2024-06-11 19:09:19,796 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:19.796616548Z 2024-06-11 19:09:19,796 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/projekt-kopieren +2024-06-11T19:09:19.796618132Z 2024-06-11 19:09:19,796 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/projekt-kopieren +2024-06-11T19:09:19.956292674Z 2024-06-11 19:09:19,956 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.16 seconds +2024-06-11T19:09:19.969269174Z 2024-06-11 19:09:19,968 - Milvus_DB - INFO - deleting all entities related to document 976690891545378817 +2024-06-11T19:09:19.978541340Z 2024-06-11 19:09:19,978 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:20.004141715Z 2024-06-11 19:09:20,004 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:20.004702215Z 2024-06-11 19:09:20,004 - root - INFO - splitting the entity +2024-06-11T19:09:20.004716840Z 2024-06-11 19:09:20,004 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:20.004718674Z 2024-06-11 19:09:20,004 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/projekt-kopieren +2024-06-11T19:09:20.004972549Z 2024-06-11 19:09:20,004 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:09:20.613037924Z 2024-06-11 19:09:20,612 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:09:23.651929592Z 2024-06-11 19:09:23,648 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:09:23.662967759Z 2024-06-11 19:09:23,662 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132963.059161758,0): "sql txn" meta={id=8d1296ba key=/Table/121/1/976691408577331201/0 iso=Serializable pri=0.08167517 epo=0 ts=1718132963.245123675,1 min=1718132960.016921049,0 seq=1} lock=true stat=PENDING rts=1718132960.016921049,0 wto=false gul=1718132960.516921049,0 +2024-06-11T19:09:23.662987300Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:23.662991592Z +2024-06-11T19:09:23.662994384Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:23.662997259Z 2024-06-11 19:09:23,662 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:23.672721259Z 2024-06-11 19:09:23,671 - __main__ - INFO - 1 +2024-06-11T19:09:23.674368050Z 2024-06-11 19:09:23,673 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:23.674380592Z 2024-06-11 19:09:23,673 - __main__ - INFO - ⏰⏰ total elapsed time: 3.91 seconds +2024-06-11T19:09:23.674384300Z 2024-06-11 19:09:23,673 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.813.813.1718132806282375975.195', data=b'\n/https://help.collaboard.app/de/projekt-kopieren(\x81\x80\xb0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:23.676540967Z 2024-06-11 19:09:23,675 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.815.815.1718132806285496059.193', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.815.815.1718132806285496059.193', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.817.817.1718132806288071100.191', data=b'\n?https://help.collaboard.app/de/benutzer-berechtigungen-anpassen(\x81\x80\xb8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:27.547525719Z 2024-06-11 19:09:27,546 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:27.547527302Z 2024-06-11 19:09:27,546 - __main__ - INFO - message: +2024-06-11T19:09:27.547528427Z url: "https://help.collaboard.app/de/benutzer-berechtigungen-anpassen" +2024-06-11T19:09:27.547529552Z document_id: 976690891545509889 +2024-06-11T19:09:27.547530469Z connector_id: 976341399530373121 +2024-06-11T19:09:27.547531469Z file_type: URL +2024-06-11T19:09:27.547532385Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:27.547533427Z +2024-06-11T19:09:27.547534260Z 2024-06-11 19:09:27,546 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:27.547535302Z 2024-06-11 19:09:27,546 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:27.580110927Z 2024-06-11 19:09:27,579 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:27.580125302Z 2024-06-11 19:09:27,579 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/benutzer-berechtigungen-anpassen +2024-06-11T19:09:27.580126969Z 2024-06-11 19:09:27,579 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/benutzer-berechtigungen-anpassen +2024-06-11T19:09:27.696179844Z 2024-06-11 19:09:27,696 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:09:27.768673011Z 2024-06-11 19:09:27,768 - Milvus_DB - INFO - deleting all entities related to document 976690891545509889 +2024-06-11T19:09:27.782381302Z 2024-06-11 19:09:27,782 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:27.793859927Z 2024-06-11 19:09:27,793 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:27.793865969Z 2024-06-11 19:09:27,793 - root - INFO - splitting the entity +2024-06-11T19:09:27.793867261Z 2024-06-11 19:09:27,793 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:27.793873136Z 2024-06-11 19:09:27,793 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/benutzer-berechtigungen-anpassen +2024-06-11T19:09:27.793875594Z 2024-06-11 19:09:27,793 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:09:28.368348886Z 2024-06-11 19:09:28,368 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:09:31.407836096Z 2024-06-11 19:09:31,407 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:09:31.412393096Z 2024-06-11 19:09:31,412 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132971.011571512,0): "sql txn" meta={id=cb7ddc2c key=/Table/121/1/976691433992454145/0 iso=Serializable pri=0.03031115 epo=0 ts=1718132971.279812679,1 min=1718132967.798583011,0 seq=1} lock=true stat=PENDING rts=1718132967.798583011,0 wto=false gul=1718132968.298583011,0 +2024-06-11T19:09:31.412399887Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:31.412401304Z +2024-06-11T19:09:31.412402387Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:31.412403387Z 2024-06-11 19:09:31,412 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:31.418043554Z 2024-06-11 19:09:31,417 - __main__ - INFO - 1 +2024-06-11T19:09:31.418961429Z 2024-06-11 19:09:31,418 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:31.418975471Z 2024-06-11 19:09:31,418 - __main__ - INFO - ⏰⏰ total elapsed time: 3.87 seconds +2024-06-11T19:09:31.418977012Z 2024-06-11 19:09:31,418 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.817.817.1718132806288071100.191', data=b'\n?https://help.collaboard.app/de/benutzer-berechtigungen-anpassen(\x81\x80\xb8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:31.419560512Z 2024-06-11 19:09:31,419 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.819.819.1718132806290842725.189', data=b'\n1https://help.collaboard.app/de/projekte-migrieren(\x81\x80\xbc\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:31.419565012Z 2024-06-11 19:09:31,419 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:31.419566471Z 2024-06-11 19:09:31,419 - __main__ - INFO - message: +2024-06-11T19:09:31.419567471Z url: "https://help.collaboard.app/de/projekte-migrieren" +2024-06-11T19:09:31.419568554Z document_id: 976690891545575425 +2024-06-11T19:09:31.419570221Z connector_id: 976341399530373121 +2024-06-11T19:09:31.419571096Z file_type: URL +2024-06-11T19:09:31.419579304Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:31.419580554Z +2024-06-11T19:09:31.419581387Z 2024-06-11 19:09:31,419 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:31.419582512Z 2024-06-11 19:09:31,419 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:31.453265262Z 2024-06-11 19:09:31,452 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:31.453278554Z 2024-06-11 19:09:31,453 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/projekte-migrieren +2024-06-11T19:09:31.453280054Z 2024-06-11 19:09:31,453 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/projekte-migrieren +2024-06-11T19:09:31.677359304Z 2024-06-11 19:09:31,677 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.22 seconds +2024-06-11T19:09:31.691552762Z 2024-06-11 19:09:31,691 - Milvus_DB - INFO - deleting all entities related to document 976690891545575425 +2024-06-11T19:09:31.702357554Z 2024-06-11 19:09:31,702 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:31.711549596Z 2024-06-11 19:09:31,711 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:31.711559929Z 2024-06-11 19:09:31,711 - root - INFO - splitting the entity +2024-06-11T19:09:31.711561304Z 2024-06-11 19:09:31,711 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:31.711562554Z 2024-06-11 19:09:31,711 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/projekte-migrieren +2024-06-11T19:09:31.711563762Z 2024-06-11 19:09:31,711 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:09:32.271161471Z 2024-06-11 19:09:32,271 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:09:35.310162848Z 2024-06-11 19:09:35,309 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.59 seconds +2024-06-11T19:09:35.316920139Z 2024-06-11 19:09:35,316 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718132975.049572472,0): "sql txn" meta={id=9ce46200 key=/Table/121/1/976691446779346945/0 iso=Serializable pri=0.01149313 epo=0 ts=1718132975.219786472,1 min=1718132971.716292179,0 seq=1} lock=true stat=PENDING rts=1718132971.716292179,0 wto=false gul=1718132972.216292179,0 +2024-06-11T19:09:35.316928848Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:09:35.316930514Z +2024-06-11T19:09:35.316931889Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:09:35.316933181Z 2024-06-11 19:09:35,316 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:09:35.329551306Z 2024-06-11 19:09:35,328 - __main__ - INFO - 1 +2024-06-11T19:09:35.330498264Z 2024-06-11 19:09:35,329 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:09:35.330500181Z 2024-06-11 19:09:35,329 - __main__ - INFO - ⏰⏰ total elapsed time: 3.91 seconds +2024-06-11T19:09:35.330501348Z 2024-06-11 19:09:35,329 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.819.819.1718132806290842725.189', data=b'\n1https://help.collaboard.app/de/projekte-migrieren(\x81\x80\xbc\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:09:35.330763806Z 2024-06-11 19:09:35,330 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.821.821.1718132806293078975.187', data=b'\nAhttps://help.collaboard.app/de/benutzer-von-meinem-team-entfernen(\x81\x80\xc0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:09:35.330767889Z 2024-06-11 19:09:35,330 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:09:35.330769223Z 2024-06-11 19:09:35,330 - __main__ - INFO - message: +2024-06-11T19:09:35.330770223Z url: "https://help.collaboard.app/de/benutzer-von-meinem-team-entfernen" +2024-06-11T19:09:35.330771306Z document_id: 976690891545640961 +2024-06-11T19:09:35.330772223Z connector_id: 976341399530373121 +2024-06-11T19:09:35.330773306Z file_type: URL +2024-06-11T19:09:35.330774223Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:09:35.330775223Z +2024-06-11T19:09:35.330776431Z 2024-06-11 19:09:35,330 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:09:35.330780181Z 2024-06-11 19:09:35,330 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:09:35.369554389Z 2024-06-11 19:09:35,369 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:09:35.369570973Z 2024-06-11 19:09:35,369 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/benutzer-von-meinem-team-entfernen +2024-06-11T19:09:35.369572639Z 2024-06-11 19:09:35,369 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/benutzer-von-meinem-team-entfernen +2024-06-11T19:09:35.482692139Z 2024-06-11 19:09:35,482 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:09:35.493669556Z 2024-06-11 19:09:35,493 - Milvus_DB - INFO - deleting all entities related to document 976690891545640961 +2024-06-11T19:09:35.502621056Z 2024-06-11 19:09:35,502 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:09:35.510696223Z 2024-06-11 19:09:35,510 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:09:35.510703639Z 2024-06-11 19:09:35,510 - root - INFO - splitting the entity +2024-06-11T19:09:35.510704931Z 2024-06-11 19:09:35,510 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:09:35.510954389Z 2024-06-11 19:09:35,510 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/benutzer-von-meinem-team-entfernen +2024-06-11T19:09:35.510960264Z 2024-06-11 19:09:35,510 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:09:36.058775890Z 2024-06-11 19:09:36,058 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:09:39.155009766Z 2024-06-11 19:09:39,154 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:09:39.802032891Z 2024-06-11 19:09:39,801 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.12 seconds +2024-06-11T19:09:41.418235850Z 127.0.0.1 - - [11/Jun/2024 19:09:41] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:09:41.418252975Z 2024-06-11 19:09:41,418 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:09:43.328951893Z 2024-06-11 19:09:43,328 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.17 seconds +2024-06-11T19:09:43.894899602Z 2024-06-11 19:09:43,894 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:09:46.955256436Z 2024-06-11 19:09:46,954 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:09:47.561645728Z 2024-06-11 19:09:47,561 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:09:50.644676855Z 2024-06-11 19:09:50,638 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:09:51.244610897Z 2024-06-11 19:09:51,244 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:09:54.316132482Z 2024-06-11 19:09:54,315 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:09:54.908844815Z 2024-06-11 19:09:54,908 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:09:58.029216608Z 2024-06-11 19:09:58,027 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:09:58.639882150Z 2024-06-11 19:09:58,639 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:10:01.201345152Z 2024-06-11 19:10:01,201 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.16 seconds +2024-06-11T19:10:01.880700694Z 2024-06-11 19:10:01,880 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.12 seconds +2024-06-11T19:10:04.921829153Z 2024-06-11 19:10:04,921 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:10:05.548494320Z 2024-06-11 19:10:05,548 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:10:08.599903280Z 2024-06-11 19:10:08,599 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:10:08.611882738Z 2024-06-11 19:10:08,611 - URLSemantic - INFO - ⏰ total elapsed time: 33.24 seconds +2024-06-11T19:10:08.611890738Z 2024-06-11 19:10:08,611 - URLSemantic - INFO - 📖 number of docs analyzed: 9 +2024-06-11T19:10:08.618005613Z 2024-06-11 19:10:08,617 - __main__ - INFO - 1 +2024-06-11T19:10:08.620259197Z 2024-06-11 19:10:08,619 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 9 +2024-06-11T19:10:08.620269238Z 2024-06-11 19:10:08,619 - __main__ - INFO - ⏰⏰ total elapsed time: 33.29 seconds +2024-06-11T19:10:08.620270905Z 2024-06-11 19:10:08,619 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.821.821.1718132806293078975.187', data=b'\nAhttps://help.collaboard.app/de/benutzer-von-meinem-team-entfernen(\x81\x80\xc0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:10:08.630419905Z 2024-06-11 19:10:08,630 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.823.823.1718132806295614059.185', data=b'\nBhttps://help.collaboard.app/de/benutzer-zu-meinem-team-hinzuf\xc3\xbcgen(\x81\x80\xc4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:10:08.630432738Z 2024-06-11 19:10:08,630 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:10:08.630434863Z 2024-06-11 19:10:08,630 - __main__ - INFO - message: +2024-06-11T19:10:08.630436947Z url: "https://help.collaboard.app/de/benutzer-zu-meinem-team-hinzufügen" +2024-06-11T19:10:08.630441363Z document_id: 976690891545706497 +2024-06-11T19:10:08.630443488Z connector_id: 976341399530373121 +2024-06-11T19:10:08.630444530Z file_type: URL +2024-06-11T19:10:08.630589780Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:10:08.630596447Z +2024-06-11T19:10:08.630598655Z 2024-06-11 19:10:08,630 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:10:08.630602072Z 2024-06-11 19:10:08,630 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:10:08.686493988Z 2024-06-11 19:10:08,686 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:10:08.686519488Z 2024-06-11 19:10:08,686 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/benutzer-zu-meinem-team-hinzufügen +2024-06-11T19:10:08.686524030Z 2024-06-11 19:10:08,686 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/benutzer-zu-meinem-team-hinzufügen +2024-06-11T19:10:08.817435697Z 2024-06-11 19:10:08,817 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.13 seconds +2024-06-11T19:10:08.829691447Z 2024-06-11 19:10:08,829 - Milvus_DB - INFO - deleting all entities related to document 976690891545706497 +2024-06-11T19:10:08.841299113Z 2024-06-11 19:10:08,841 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:10:08.850200155Z 2024-06-11 19:10:08,850 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:10:08.850208280Z 2024-06-11 19:10:08,850 - root - INFO - splitting the entity +2024-06-11T19:10:08.850210322Z 2024-06-11 19:10:08,850 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:10:08.850624572Z 2024-06-11 19:10:08,850 - URLSemantic - INFO - created 14 chunks for https://help.collaboard.app/de/benutzer-zu-meinem-team-hinzufügen +2024-06-11T19:10:08.850627697Z 2024-06-11 19:10:08,850 - root - INFO - saving in milvus 14 chunk(s) +2024-06-11T19:10:09.461809447Z 2024-06-11 19:10:09,461 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:10:12.495885240Z 2024-06-11 19:10:12,495 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:10:13.125662282Z 2024-06-11 19:10:13,125 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:10:16.175429367Z 2024-06-11 19:10:16,175 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:10:16.734977909Z 2024-06-11 19:10:16,734 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:10:20.322649577Z 2024-06-11 19:10:20,322 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.14 seconds +2024-06-11T19:10:20.924028244Z 2024-06-11 19:10:20,923 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:10:24.011700371Z 2024-06-11 19:10:24,011 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:10:24.599518871Z 2024-06-11 19:10:24,599 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:10:27.197120539Z 2024-06-11 19:10:27,196 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.17 seconds +2024-06-11T19:10:27.857309248Z 2024-06-11 19:10:27,856 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.13 seconds +2024-06-11T19:10:30.904298957Z 2024-06-11 19:10:30,904 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:10:31.506752124Z 2024-06-11 19:10:31,506 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:10:34.561006042Z 2024-06-11 19:10:34,559 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:10:35.176365126Z 2024-06-11 19:10:35,176 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:10:38.228339128Z 2024-06-11 19:10:38,227 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:10:38.896440169Z 2024-06-11 19:10:38,896 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.12 seconds +2024-06-11T19:10:41.530768004Z 127.0.0.1 - - [11/Jun/2024 19:10:41] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:10:41.530782879Z 2024-06-11 19:10:41,530 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:10:42.014588296Z 2024-06-11 19:10:42,014 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.77 seconds +2024-06-11T19:10:42.797827255Z 2024-06-11 19:10:42,797 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.25 seconds +2024-06-11T19:10:45.865847839Z 2024-06-11 19:10:45,865 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.84 seconds +2024-06-11T19:10:46.496384506Z 2024-06-11 19:10:46,496 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:10:49.547068841Z 2024-06-11 19:10:49,546 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:10:50.141488633Z 2024-06-11 19:10:50,141 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:10:53.216884010Z 2024-06-11 19:10:53,216 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:10:53.900819510Z 2024-06-11 19:10:53,900 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:10:56.965944178Z 2024-06-11 19:10:56,965 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.72 seconds +2024-06-11T19:10:57.543093720Z 2024-06-11 19:10:57,542 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:11:00.608843722Z 2024-06-11 19:11:00,608 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:11:00.631345305Z 2024-06-11 19:11:00,630 - URLSemantic - INFO - ⏰ total elapsed time: 51.94 seconds +2024-06-11T19:11:00.631365888Z 2024-06-11 19:11:00,630 - URLSemantic - INFO - 📖 number of docs analyzed: 14 +2024-06-11T19:11:00.641145722Z 2024-06-11 19:11:00,639 - __main__ - INFO - 1 +2024-06-11T19:11:00.642921430Z 2024-06-11 19:11:00,641 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 14 +2024-06-11T19:11:00.642934597Z 2024-06-11 19:11:00,641 - __main__ - INFO - ⏰⏰ total elapsed time: 52.01 seconds +2024-06-11T19:11:00.642937888Z 2024-06-11 19:11:00,641 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.823.823.1718132806295614059.185', data=b'\nBhttps://help.collaboard.app/de/benutzer-zu-meinem-team-hinzuf\xc3\xbcgen(\x81\x80\xc4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:00.653541388Z 2024-06-11 19:11:00,653 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.825.825.1718132806298023600.183', data=b'\nKhttps://help.collaboard.app/de/fuege-einem-abonnement-mehrere-manager-hinzu(\x81\x80\xc8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:00.653569305Z 2024-06-11 19:11:00,653 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:00.653571930Z 2024-06-11 19:11:00,653 - __main__ - INFO - message: +2024-06-11T19:11:00.653573513Z url: "https://help.collaboard.app/de/fuege-einem-abonnement-mehrere-manager-hinzu" +2024-06-11T19:11:00.653575097Z document_id: 976690891545772033 +2024-06-11T19:11:00.653576430Z connector_id: 976341399530373121 +2024-06-11T19:11:00.653577638Z file_type: URL +2024-06-11T19:11:00.653578847Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:00.653580263Z +2024-06-11T19:11:00.653581472Z 2024-06-11 19:11:00,653 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:00.653582888Z 2024-06-11 19:11:00,653 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:00.690942847Z 2024-06-11 19:11:00,690 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:00.690969472Z 2024-06-11 19:11:00,690 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/fuege-einem-abonnement-mehrere-manager-hinzu +2024-06-11T19:11:00.690971555Z 2024-06-11 19:11:00,690 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/fuege-einem-abonnement-mehrere-manager-hinzu +2024-06-11T19:11:00.774262138Z 2024-06-11 19:11:00,774 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.08 seconds +2024-06-11T19:11:00.786132263Z 2024-06-11 19:11:00,785 - Milvus_DB - INFO - deleting all entities related to document 976690891545772033 +2024-06-11T19:11:00.796649347Z 2024-06-11 19:11:00,796 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:00.806207472Z 2024-06-11 19:11:00,806 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:00.806213722Z 2024-06-11 19:11:00,806 - root - INFO - splitting the entity +2024-06-11T19:11:00.806215138Z 2024-06-11 19:11:00,806 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:00.806444305Z 2024-06-11 19:11:00,806 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/fuege-einem-abonnement-mehrere-manager-hinzu +2024-06-11T19:11:00.806448055Z 2024-06-11 19:11:00,806 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:11:01.393074472Z 2024-06-11 19:11:01,392 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:11:04.443360807Z 2024-06-11 19:11:04,442 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:11:04.452927348Z 2024-06-11 19:11:04,452 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133062.033783514,0): "sql txn" meta={id=482241c2 key=/Table/121/1/976691738853965825/0 iso=Serializable pri=0.01894692 epo=0 ts=1718133062.251952431,1 min=1718133060.811616305,0 seq=1} lock=true stat=PENDING rts=1718133060.811616305,0 wto=false gul=1718133061.311616305,0 +2024-06-11T19:11:04.452954348Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:04.452959848Z +2024-06-11T19:11:04.452963390Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:04.452965598Z 2024-06-11 19:11:04,452 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:04.462679390Z 2024-06-11 19:11:04,462 - __main__ - INFO - 1 +2024-06-11T19:11:04.465741098Z 2024-06-11 19:11:04,465 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:04.465751182Z 2024-06-11 19:11:04,465 - __main__ - INFO - ⏰⏰ total elapsed time: 3.81 seconds +2024-06-11T19:11:04.465753682Z 2024-06-11 19:11:04,465 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.825.825.1718132806298023600.183', data=b'\nKhttps://help.collaboard.app/de/fuege-einem-abonnement-mehrere-manager-hinzu(\x81\x80\xc8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:04.478073640Z 2024-06-11 19:11:04,477 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.827.827.1718132806300291475.181', data=b'\nfhttps://help.collaboard.app/de/wie-kann-ich-die-e-mail-adresse-eines-benutzers-aus-meinem-team-\xc3\xa4ndern(\x81\x80\xcc\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:04.478097890Z 2024-06-11 19:11:04,477 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:04.478103473Z 2024-06-11 19:11:04,477 - __main__ - INFO - message: +2024-06-11T19:11:04.478105390Z url: "https://help.collaboard.app/de/wie-kann-ich-die-e-mail-adresse-eines-benutzers-aus-meinem-team-ändern" +2024-06-11T19:11:04.478108390Z document_id: 976690891545837569 +2024-06-11T19:11:04.478112140Z connector_id: 976341399530373121 +2024-06-11T19:11:04.478113432Z file_type: URL +2024-06-11T19:11:04.478114598Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:04.478118182Z +2024-06-11T19:11:04.478120890Z 2024-06-11 19:11:04,477 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:04.478122348Z 2024-06-11 19:11:04,477 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:04.521275432Z 2024-06-11 19:11:04,520 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:04.521294598Z 2024-06-11 19:11:04,520 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-kann-ich-die-e-mail-adresse-eines-benutzers-aus-meinem-team-ändern +2024-06-11T19:11:04.521296723Z 2024-06-11 19:11:04,520 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-kann-ich-die-e-mail-adresse-eines-benutzers-aus-meinem-team-ändern +2024-06-11T19:11:04.701342182Z 2024-06-11 19:11:04,701 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.18 seconds +2024-06-11T19:11:04.719736515Z 2024-06-11 19:11:04,719 - Milvus_DB - INFO - deleting all entities related to document 976690891545837569 +2024-06-11T19:11:04.734782682Z 2024-06-11 19:11:04,734 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:11:04.745315932Z 2024-06-11 19:11:04,745 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:04.745330265Z 2024-06-11 19:11:04,745 - root - INFO - splitting the entity +2024-06-11T19:11:04.745331723Z 2024-06-11 19:11:04,745 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:04.745681307Z 2024-06-11 19:11:04,745 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/wie-kann-ich-die-e-mail-adresse-eines-benutzers-aus-meinem-team-ändern +2024-06-11T19:11:04.745690473Z 2024-06-11 19:11:04,745 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:11:05.340060890Z 2024-06-11 19:11:05,339 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:11:08.423079225Z 2024-06-11 19:11:08,422 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:11:08.438827100Z 2024-06-11 19:11:08,438 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133066.116067516,0): "sql txn" meta={id=b5e86aaa key=/Table/121/1/976691751896547329/0 iso=Serializable pri=0.00541885 epo=0 ts=1718133066.347033974,1 min=1718133064.750614723,0 seq=1} lock=true stat=PENDING rts=1718133064.750614723,0 wto=false gul=1718133065.250614723,0 +2024-06-11T19:11:08.438847100Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:08.438850017Z +2024-06-11T19:11:08.438852100Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:08.438854475Z 2024-06-11 19:11:08,438 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:08.446378934Z 2024-06-11 19:11:08,445 - __main__ - INFO - 1 +2024-06-11T19:11:08.447370642Z 2024-06-11 19:11:08,446 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:08.447381225Z 2024-06-11 19:11:08,446 - __main__ - INFO - ⏰⏰ total elapsed time: 3.97 seconds +2024-06-11T19:11:08.447384892Z 2024-06-11 19:11:08,446 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.827.827.1718132806300291475.181', data=b'\nfhttps://help.collaboard.app/de/wie-kann-ich-die-e-mail-adresse-eines-benutzers-aus-meinem-team-\xc3\xa4ndern(\x81\x80\xcc\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:08.457150559Z 2024-06-11 19:11:08,456 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.829.829.1718132806302631225.179', data=b'\n;https://help.collaboard.app/administration/manage-your-team(\x81\x80\xd0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:08.457169059Z 2024-06-11 19:11:08,456 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:08.457171392Z 2024-06-11 19:11:08,456 - __main__ - INFO - message: +2024-06-11T19:11:08.457172850Z url: "https://help.collaboard.app/administration/manage-your-team" +2024-06-11T19:11:08.457174350Z document_id: 976690891545903105 +2024-06-11T19:11:08.457175600Z connector_id: 976341399530373121 +2024-06-11T19:11:08.457176892Z file_type: URL +2024-06-11T19:11:08.457178017Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:08.457179350Z +2024-06-11T19:11:08.457180475Z 2024-06-11 19:11:08,456 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:08.457181767Z 2024-06-11 19:11:08,456 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:08.492000059Z 2024-06-11 19:11:08,491 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:08.492014642Z 2024-06-11 19:11:08,491 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/administration/manage-your-team +2024-06-11T19:11:08.492016600Z 2024-06-11 19:11:08,491 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/administration/manage-your-team +2024-06-11T19:11:08.872202600Z 2024-06-11 19:11:08,871 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.38 seconds +2024-06-11T19:11:08.890745684Z 2024-06-11 19:11:08,890 - Milvus_DB - INFO - deleting all entities related to document 976690891545903105 +2024-06-11T19:11:08.906309017Z 2024-06-11 19:11:08,906 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:11:08.916045559Z 2024-06-11 19:11:08,915 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:08.916051725Z 2024-06-11 19:11:08,915 - root - INFO - splitting the entity +2024-06-11T19:11:08.916053142Z 2024-06-11 19:11:08,915 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:08.916206475Z 2024-06-11 19:11:08,916 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/administration/manage-your-team +2024-06-11T19:11:08.916209392Z 2024-06-11 19:11:08,916 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:11:09.501675684Z 2024-06-11 19:11:09,501 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:11:12.547388686Z 2024-06-11 19:11:12,546 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:11:12.568566561Z 2024-06-11 19:11:12,567 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133070.157351351,0): "sql txn" meta={id=721ca314 key=/Table/121/1/976691765405548545/0 iso=Serializable pri=0.00193771 epo=0 ts=1718133070.775469518,1 min=1718133068.921418975,0 seq=1} lock=true stat=PENDING rts=1718133068.921418975,0 wto=false gul=1718133069.421418975,0 +2024-06-11T19:11:12.568586686Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:12.568588352Z +2024-06-11T19:11:12.568589311Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:12.568590436Z 2024-06-11 19:11:12,568 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:12.605987561Z 2024-06-11 19:11:12,605 - __main__ - INFO - 1 +2024-06-11T19:11:12.607389269Z 2024-06-11 19:11:12,606 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:12.607395852Z 2024-06-11 19:11:12,606 - __main__ - INFO - ⏰⏰ total elapsed time: 4.15 seconds +2024-06-11T19:11:12.607397436Z 2024-06-11 19:11:12,606 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.829.829.1718132806302631225.179', data=b'\n;https://help.collaboard.app/administration/manage-your-team(\x81\x80\xd0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:12.627640394Z 2024-06-11 19:11:12,626 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.831.831.1718132806304906392.177', data=b'\nXhttps://help.collaboard.app/de/wie-kann-ich-meinem-team-in-collaboard-nutzer-hinzuf\xc3\xbcgen(\x81\x80\xd4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:12.627647727Z 2024-06-11 19:11:12,626 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:12.627649269Z 2024-06-11 19:11:12,626 - __main__ - INFO - message: +2024-06-11T19:11:12.627650436Z url: "https://help.collaboard.app/de/wie-kann-ich-meinem-team-in-collaboard-nutzer-hinzufügen" +2024-06-11T19:11:12.627651602Z document_id: 976690891545968641 +2024-06-11T19:11:12.627652561Z connector_id: 976341399530373121 +2024-06-11T19:11:12.627653477Z file_type: URL +2024-06-11T19:11:12.627654352Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:12.627655352Z +2024-06-11T19:11:12.627656186Z 2024-06-11 19:11:12,626 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:12.627657186Z 2024-06-11 19:11:12,626 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:12.708661102Z 2024-06-11 19:11:12,708 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:12.708676602Z 2024-06-11 19:11:12,708 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-kann-ich-meinem-team-in-collaboard-nutzer-hinzufügen +2024-06-11T19:11:12.708678561Z 2024-06-11 19:11:12,708 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-kann-ich-meinem-team-in-collaboard-nutzer-hinzufügen +2024-06-11T19:11:12.870195602Z 2024-06-11 19:11:12,870 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.16 seconds +2024-06-11T19:11:12.883337936Z 2024-06-11 19:11:12,883 - Milvus_DB - INFO - deleting all entities related to document 976690891545968641 +2024-06-11T19:11:12.894370561Z 2024-06-11 19:11:12,894 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:12.904089394Z 2024-06-11 19:11:12,903 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:12.904095144Z 2024-06-11 19:11:12,903 - root - INFO - splitting the entity +2024-06-11T19:11:12.904096311Z 2024-06-11 19:11:12,903 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:12.904650061Z 2024-06-11 19:11:12,904 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/wie-kann-ich-meinem-team-in-collaboard-nutzer-hinzufügen +2024-06-11T19:11:12.904658727Z 2024-06-11 19:11:12,904 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:11:13.517240561Z 2024-06-11 19:11:13,517 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:11:16.573452146Z 2024-06-11 19:11:16,572 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:11:16.580029437Z 2024-06-11 19:11:16,579 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133074.619381312,0): "sql txn" meta={id=a86f789c key=/Table/121/1/976691778599878657/0 iso=Serializable pri=0.00611884 epo=0 ts=1718133074.809286687,1 min=1718133072.909673019,0 seq=1} lock=true stat=PENDING rts=1718133072.909673019,0 wto=false gul=1718133073.409673019,0 +2024-06-11T19:11:16.580045021Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:16.580046812Z +2024-06-11T19:11:16.580047979Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:16.580049187Z 2024-06-11 19:11:16,579 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:16.586888271Z 2024-06-11 19:11:16,586 - __main__ - INFO - 1 +2024-06-11T19:11:16.589063312Z 2024-06-11 19:11:16,588 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:16.589069312Z 2024-06-11 19:11:16,588 - __main__ - INFO - ⏰⏰ total elapsed time: 3.96 seconds +2024-06-11T19:11:16.589071479Z 2024-06-11 19:11:16,588 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.831.831.1718132806304906392.177', data=b'\nXhttps://help.collaboard.app/de/wie-kann-ich-meinem-team-in-collaboard-nutzer-hinzuf\xc3\xbcgen(\x81\x80\xd4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:16.597420604Z 2024-06-11 19:11:16,597 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.833.833.1718132806308333934.175', data=b'\n,https://help.collaboard.app/export-user-list(\x81\x80\xd8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:16.597428146Z 2024-06-11 19:11:16,597 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:16.597429771Z 2024-06-11 19:11:16,597 - __main__ - INFO - message: +2024-06-11T19:11:16.597430896Z url: "https://help.collaboard.app/export-user-list" +2024-06-11T19:11:16.597432021Z document_id: 976690891546034177 +2024-06-11T19:11:16.597432937Z connector_id: 976341399530373121 +2024-06-11T19:11:16.597433812Z file_type: URL +2024-06-11T19:11:16.597434687Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:16.597435687Z +2024-06-11T19:11:16.597436521Z 2024-06-11 19:11:16,597 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:16.597437521Z 2024-06-11 19:11:16,597 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:16.630002604Z 2024-06-11 19:11:16,629 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:16.630050979Z 2024-06-11 19:11:16,629 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/export-user-list +2024-06-11T19:11:16.630056396Z 2024-06-11 19:11:16,629 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/export-user-list +2024-06-11T19:11:16.733674354Z 2024-06-11 19:11:16,733 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:11:16.744386188Z 2024-06-11 19:11:16,744 - Milvus_DB - INFO - deleting all entities related to document 976690891546034177 +2024-06-11T19:11:16.755743979Z 2024-06-11 19:11:16,755 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:16.765315688Z 2024-06-11 19:11:16,765 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:16.765323313Z 2024-06-11 19:11:16,765 - root - INFO - splitting the entity +2024-06-11T19:11:16.765324729Z 2024-06-11 19:11:16,765 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:16.765665104Z 2024-06-11 19:11:16,765 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/export-user-list +2024-06-11T19:11:16.765672313Z 2024-06-11 19:11:16,765 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:11:17.353569271Z 2024-06-11 19:11:17,353 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:11:20.396945106Z 2024-06-11 19:11:20,396 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:11:20.405238981Z 2024-06-11 19:11:20,404 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133078.618673730,0): "sql txn" meta={id=6116b009 key=/Table/121/1/976691791129772033/0 iso=Serializable pri=0.01376662 epo=0 ts=1718133078.771826272,1 min=1718133076.769878313,0 seq=1} lock=true stat=PENDING rts=1718133076.769878313,0 wto=false gul=1718133077.269878313,0 +2024-06-11T19:11:20.405251273Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:20.405255189Z +2024-06-11T19:11:20.405256898Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:20.405258314Z 2024-06-11 19:11:20,405 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:20.411307814Z 2024-06-11 19:11:20,410 - __main__ - INFO - 1 +2024-06-11T19:11:20.412352939Z 2024-06-11 19:11:20,412 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:20.412358773Z 2024-06-11 19:11:20,412 - __main__ - INFO - ⏰⏰ total elapsed time: 3.81 seconds +2024-06-11T19:11:20.412360689Z 2024-06-11 19:11:20,412 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.833.833.1718132806308333934.175', data=b'\n,https://help.collaboard.app/export-user-list(\x81\x80\xd8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:20.413533314Z 2024-06-11 19:11:20,413 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.835.835.1718132806310410559.173', data=b'\n&https://help.collaboard.app/how-to-add(\x81\x80\xdc\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:20.413545564Z 2024-06-11 19:11:20,413 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:20.413547814Z 2024-06-11 19:11:20,413 - __main__ - INFO - message: +2024-06-11T19:11:20.413549314Z url: "https://help.collaboard.app/how-to-add" +2024-06-11T19:11:20.413550689Z document_id: 976690891546099713 +2024-06-11T19:11:20.413551814Z connector_id: 976341399530373121 +2024-06-11T19:11:20.413552939Z file_type: URL +2024-06-11T19:11:20.413554356Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:20.413555648Z +2024-06-11T19:11:20.413556731Z 2024-06-11 19:11:20,413 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:20.413558064Z 2024-06-11 19:11:20,413 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:20.450035773Z 2024-06-11 19:11:20,449 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:20.450050898Z 2024-06-11 19:11:20,449 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/how-to-add +2024-06-11T19:11:20.450052231Z 2024-06-11 19:11:20,449 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/how-to-add +2024-06-11T19:11:20.594322023Z 2024-06-11 19:11:20,594 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.14 seconds +2024-06-11T19:11:20.605893148Z 2024-06-11 19:11:20,605 - Milvus_DB - INFO - deleting all entities related to document 976690891546099713 +2024-06-11T19:11:20.617062648Z 2024-06-11 19:11:20,616 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:20.628521398Z 2024-06-11 19:11:20,628 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:20.628528939Z 2024-06-11 19:11:20,628 - root - INFO - splitting the entity +2024-06-11T19:11:20.628530231Z 2024-06-11 19:11:20,628 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:20.629000106Z 2024-06-11 19:11:20,628 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/how-to-add +2024-06-11T19:11:20.629003731Z 2024-06-11 19:11:20,628 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:11:21.253416148Z 2024-06-11 19:11:21,253 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:11:23.804130441Z 2024-06-11 19:11:23,803 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.17 seconds +2024-06-11T19:11:23.810008858Z 2024-06-11 19:11:23,809 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133082.734932107,0): "sql txn" meta={id=b54c767a key=/Table/121/1/976691802291929089/0 iso=Serializable pri=0.00345707 epo=0 ts=1718133082.929581607,1 min=1718133080.633831148,0 seq=1} lock=true stat=PENDING rts=1718133080.633831148,0 wto=false gul=1718133081.133831148,0 +2024-06-11T19:11:23.810017608Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:23.810019274Z +2024-06-11T19:11:23.810020524Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:23.810021774Z 2024-06-11 19:11:23,809 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:23.815435524Z 2024-06-11 19:11:23,815 - __main__ - INFO - 1 +2024-06-11T19:11:23.817163691Z 2024-06-11 19:11:23,816 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:23.817169816Z 2024-06-11 19:11:23,816 - __main__ - INFO - ⏰⏰ total elapsed time: 3.40 seconds +2024-06-11T19:11:23.817171399Z 2024-06-11 19:11:23,816 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.835.835.1718132806310410559.173', data=b'\n&https://help.collaboard.app/how-to-add(\x81\x80\xdc\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:23.817271816Z 2024-06-11 19:11:23,817 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.837.837.1718132806312626684.171', data=b'\n:https://help.collaboard.app/how-to-add-users-to-your-space(\x81\x80\xe0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:23.817278274Z 2024-06-11 19:11:23,817 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:23.817279649Z 2024-06-11 19:11:23,817 - __main__ - INFO - message: +2024-06-11T19:11:23.817280649Z url: "https://help.collaboard.app/how-to-add-users-to-your-space" +2024-06-11T19:11:23.817281774Z document_id: 976690891546165249 +2024-06-11T19:11:23.817282691Z connector_id: 976341399530373121 +2024-06-11T19:11:23.817283524Z file_type: URL +2024-06-11T19:11:23.817284399Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:23.817285441Z +2024-06-11T19:11:23.817286316Z 2024-06-11 19:11:23,817 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:23.817287608Z 2024-06-11 19:11:23,817 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:23.853340733Z 2024-06-11 19:11:23,852 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:23.853348149Z 2024-06-11 19:11:23,852 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/how-to-add-users-to-your-space +2024-06-11T19:11:23.853349524Z 2024-06-11 19:11:23,853 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/how-to-add-users-to-your-space +2024-06-11T19:11:23.965018108Z 2024-06-11 19:11:23,964 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:11:23.976600024Z 2024-06-11 19:11:23,976 - Milvus_DB - INFO - deleting all entities related to document 976690891546165249 +2024-06-11T19:11:23.987471274Z 2024-06-11 19:11:23,987 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:23.996920983Z 2024-06-11 19:11:23,996 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:23.996928149Z 2024-06-11 19:11:23,996 - root - INFO - splitting the entity +2024-06-11T19:11:23.996929524Z 2024-06-11 19:11:23,996 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:23.997238608Z 2024-06-11 19:11:23,996 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/how-to-add-users-to-your-space +2024-06-11T19:11:23.997243524Z 2024-06-11 19:11:23,997 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:11:24.606423608Z 2024-06-11 19:11:24,606 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:11:28.187406818Z 2024-06-11 19:11:28,186 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.18 seconds +2024-06-11T19:11:28.208376693Z 2024-06-11 19:11:28,208 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133086.730148651,0): "sql txn" meta={id=31d7ab89 key=/Table/121/1/976691816664760321/0 iso=Serializable pri=0.00512400 epo=0 ts=1718133086.885783734,1 min=1718133084.001258066,0 seq=1} lock=true stat=PENDING rts=1718133084.001258066,0 wto=false gul=1718133084.501258066,0 +2024-06-11T19:11:28.208450276Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:28.208459068Z +2024-06-11T19:11:28.208501776Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:28.208561318Z 2024-06-11 19:11:28,208 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:28.221493568Z 2024-06-11 19:11:28,220 - __main__ - INFO - 1 +2024-06-11T19:11:28.223773026Z 2024-06-11 19:11:28,223 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:28.223783526Z 2024-06-11 19:11:28,223 - __main__ - INFO - ⏰⏰ total elapsed time: 4.41 seconds +2024-06-11T19:11:28.223786568Z 2024-06-11 19:11:28,223 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.837.837.1718132806312626684.171', data=b'\n:https://help.collaboard.app/how-to-add-users-to-your-space(\x81\x80\xe0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:28.227486901Z 2024-06-11 19:11:28,226 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.839.839.1718132806316130475.169', data=b'\n?https://help.collaboard.app/how-to-create-a-space-in-collaboard(\x81\x80\xe4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:28.227619443Z 2024-06-11 19:11:28,227 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:28.227627360Z 2024-06-11 19:11:28,227 - __main__ - INFO - message: +2024-06-11T19:11:28.227629735Z url: "https://help.collaboard.app/how-to-create-a-space-in-collaboard" +2024-06-11T19:11:28.227631901Z document_id: 976690891546230785 +2024-06-11T19:11:28.227633901Z connector_id: 976341399530373121 +2024-06-11T19:11:28.227635651Z file_type: URL +2024-06-11T19:11:28.227637318Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:28.227639276Z +2024-06-11T19:11:28.227640943Z 2024-06-11 19:11:28,227 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:28.227642901Z 2024-06-11 19:11:28,227 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:28.270625568Z 2024-06-11 19:11:28,270 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:28.270642943Z 2024-06-11 19:11:28,270 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/how-to-create-a-space-in-collaboard +2024-06-11T19:11:28.270644526Z 2024-06-11 19:11:28,270 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/how-to-create-a-space-in-collaboard +2024-06-11T19:11:28.351703526Z 2024-06-11 19:11:28,351 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.08 seconds +2024-06-11T19:11:28.363868360Z 2024-06-11 19:11:28,363 - Milvus_DB - INFO - deleting all entities related to document 976690891546230785 +2024-06-11T19:11:28.375670735Z 2024-06-11 19:11:28,375 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:28.385333193Z 2024-06-11 19:11:28,385 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:28.385339901Z 2024-06-11 19:11:28,385 - root - INFO - splitting the entity +2024-06-11T19:11:28.385341276Z 2024-06-11 19:11:28,385 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:28.385675526Z 2024-06-11 19:11:28,385 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/how-to-create-a-space-in-collaboard +2024-06-11T19:11:28.385680360Z 2024-06-11 19:11:28,385 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:11:28.975287318Z 2024-06-11 19:11:28,975 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:11:32.051708362Z 2024-06-11 19:11:32,050 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:11:32.057860862Z 2024-06-11 19:11:32,057 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133090.678195986,0): "sql txn" meta={id=5da8e3e7 key=/Table/121/1/976691829316812801/0 iso=Serializable pri=0.01856401 epo=0 ts=1718133090.864382028,1 min=1718133088.389992151,0 seq=1} lock=true stat=PENDING rts=1718133088.389992151,0 wto=false gul=1718133088.889992151,0 +2024-06-11T19:11:32.057872778Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:32.057874320Z +2024-06-11T19:11:32.057875320Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:32.057876528Z 2024-06-11 19:11:32,057 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:32.065313653Z 2024-06-11 19:11:32,064 - __main__ - INFO - 1 +2024-06-11T19:11:32.066392737Z 2024-06-11 19:11:32,066 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:32.066397945Z 2024-06-11 19:11:32,066 - __main__ - INFO - ⏰⏰ total elapsed time: 3.84 seconds +2024-06-11T19:11:32.066399403Z 2024-06-11 19:11:32,066 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.839.839.1718132806316130475.169', data=b'\n?https://help.collaboard.app/how-to-create-a-space-in-collaboard(\x81\x80\xe4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:32.066990320Z 2024-06-11 19:11:32,066 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.841.841.1718132806318879559.167', data=b'\n>https://help.collaboard.app/how-to-add-and-change-space-admins(\x81\x80\xe8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:32.066996028Z 2024-06-11 19:11:32,066 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:32.066997403Z 2024-06-11 19:11:32,066 - __main__ - INFO - message: +2024-06-11T19:11:32.066998403Z url: "https://help.collaboard.app/how-to-add-and-change-space-admins" +2024-06-11T19:11:32.067000195Z document_id: 976690891546296321 +2024-06-11T19:11:32.067001112Z connector_id: 976341399530373121 +2024-06-11T19:11:32.067003362Z file_type: URL +2024-06-11T19:11:32.067004278Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:32.067005945Z +2024-06-11T19:11:32.067006820Z 2024-06-11 19:11:32,066 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:32.067007862Z 2024-06-11 19:11:32,066 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:32.116078028Z 2024-06-11 19:11:32,115 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:32.116103362Z 2024-06-11 19:11:32,115 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/how-to-add-and-change-space-admins +2024-06-11T19:11:32.116105112Z 2024-06-11 19:11:32,115 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/how-to-add-and-change-space-admins +2024-06-11T19:11:32.232826945Z 2024-06-11 19:11:32,232 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:11:32.245657903Z 2024-06-11 19:11:32,245 - Milvus_DB - INFO - deleting all entities related to document 976690891546296321 +2024-06-11T19:11:32.257395945Z 2024-06-11 19:11:32,257 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:32.266819612Z 2024-06-11 19:11:32,266 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:32.266829737Z 2024-06-11 19:11:32,266 - root - INFO - splitting the entity +2024-06-11T19:11:32.266831070Z 2024-06-11 19:11:32,266 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:32.267042112Z 2024-06-11 19:11:32,266 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/how-to-add-and-change-space-admins +2024-06-11T19:11:32.267045737Z 2024-06-11 19:11:32,266 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:11:32.820343695Z 2024-06-11 19:11:32,820 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:11:35.859468155Z 2024-06-11 19:11:35,859 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.59 seconds +2024-06-11T19:11:35.865397738Z 2024-06-11 19:11:35,865 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133094.663778071,0): "sql txn" meta={id=6400ac1f key=/Table/121/1/976691841794539521/0 iso=Serializable pri=0.00561001 epo=0 ts=1718133094.820757654,1 min=1718133092.271414903,0 seq=1} lock=true stat=PENDING rts=1718133092.271414903,0 wto=false gul=1718133092.771414903,0 +2024-06-11T19:11:35.865406947Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:35.865409030Z +2024-06-11T19:11:35.865410447Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:35.865411863Z 2024-06-11 19:11:35,865 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:35.870590780Z 2024-06-11 19:11:35,870 - __main__ - INFO - 1 +2024-06-11T19:11:35.871504988Z 2024-06-11 19:11:35,870 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:35.871509238Z 2024-06-11 19:11:35,870 - __main__ - INFO - ⏰⏰ total elapsed time: 3.80 seconds +2024-06-11T19:11:35.871510572Z 2024-06-11 19:11:35,870 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.841.841.1718132806318879559.167', data=b'\n>https://help.collaboard.app/how-to-add-and-change-space-admins(\x81\x80\xe8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:35.873571738Z 2024-06-11 19:11:35,871 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.843.843.1718132806321525100.165', data=b'\n7https://help.collaboard.app/microsoft-teams-integration(\x81\x80\xec\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:35.873578488Z 2024-06-11 19:11:35,871 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:35.873580030Z 2024-06-11 19:11:35,871 - __main__ - INFO - message: +2024-06-11T19:11:35.873581280Z url: "https://help.collaboard.app/microsoft-teams-integration" +2024-06-11T19:11:35.873582447Z document_id: 976690891546361857 +2024-06-11T19:11:35.873583447Z connector_id: 976341399530373121 +2024-06-11T19:11:35.873585030Z file_type: URL +2024-06-11T19:11:35.873585947Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:35.873587030Z +2024-06-11T19:11:35.873587905Z 2024-06-11 19:11:35,871 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:35.873595405Z 2024-06-11 19:11:35,871 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:35.905582405Z 2024-06-11 19:11:35,905 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:35.905595530Z 2024-06-11 19:11:35,905 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/microsoft-teams-integration +2024-06-11T19:11:35.905597113Z 2024-06-11 19:11:35,905 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/microsoft-teams-integration +2024-06-11T19:11:36.032790113Z 2024-06-11 19:11:36,032 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.13 seconds +2024-06-11T19:11:36.046185072Z 2024-06-11 19:11:36,045 - Milvus_DB - INFO - deleting all entities related to document 976690891546361857 +2024-06-11T19:11:36.057781947Z 2024-06-11 19:11:36,057 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:36.068436780Z 2024-06-11 19:11:36,068 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:36.068450363Z 2024-06-11 19:11:36,068 - root - INFO - splitting the entity +2024-06-11T19:11:36.068452447Z 2024-06-11 19:11:36,068 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:36.068783988Z 2024-06-11 19:11:36,068 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/microsoft-teams-integration +2024-06-11T19:11:36.068787822Z 2024-06-11 19:11:36,068 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:11:36.628214322Z 2024-06-11 19:11:36,627 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:11:40.172244949Z 2024-06-11 19:11:40,171 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.10 seconds +2024-06-11T19:11:40.177809199Z 2024-06-11 19:11:40,177 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133098.654112990,0): "sql txn" meta={id=8f207b7b key=/Table/121/1/976691855926820865/0 iso=Serializable pri=0.01621046 epo=0 ts=1718133098.896451615,1 min=1718133096.073297572,0 seq=1} lock=true stat=PENDING rts=1718133096.073297572,0 wto=false gul=1718133096.573297572,0 +2024-06-11T19:11:40.177818907Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:40.177820615Z +2024-06-11T19:11:40.177821949Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:40.177823199Z 2024-06-11 19:11:40,177 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:40.183562157Z 2024-06-11 19:11:40,182 - __main__ - INFO - 1 +2024-06-11T19:11:40.185008699Z 2024-06-11 19:11:40,184 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:40.185014949Z 2024-06-11 19:11:40,184 - __main__ - INFO - ⏰⏰ total elapsed time: 4.31 seconds +2024-06-11T19:11:40.185022699Z 2024-06-11 19:11:40,184 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.843.843.1718132806321525100.165', data=b'\n7https://help.collaboard.app/microsoft-teams-integration(\x81\x80\xec\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:40.185493865Z 2024-06-11 19:11:40,185 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.845.845.1718132806324427975.163', data=b'\n7https://help.collaboard.app/administration/integrations(\x81\x80\xf0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:40.185497907Z 2024-06-11 19:11:40,185 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:40.185499115Z 2024-06-11 19:11:40,185 - __main__ - INFO - message: +2024-06-11T19:11:40.185500115Z url: "https://help.collaboard.app/administration/integrations" +2024-06-11T19:11:40.185501240Z document_id: 976690891546427393 +2024-06-11T19:11:40.185502282Z connector_id: 976341399530373121 +2024-06-11T19:11:40.185503199Z file_type: URL +2024-06-11T19:11:40.185504074Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:40.185505115Z +2024-06-11T19:11:40.185505990Z 2024-06-11 19:11:40,185 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:40.185506990Z 2024-06-11 19:11:40,185 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:40.220455240Z 2024-06-11 19:11:40,220 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:40.220468782Z 2024-06-11 19:11:40,220 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/administration/integrations +2024-06-11T19:11:40.220470532Z 2024-06-11 19:11:40,220 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/administration/integrations +2024-06-11T19:11:40.666967991Z 2024-06-11 19:11:40,666 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.45 seconds +2024-06-11T19:11:40.680013616Z 2024-06-11 19:11:40,679 - Milvus_DB - INFO - deleting all entities related to document 976690891546427393 +2024-06-11T19:11:40.694687491Z 2024-06-11 19:11:40,694 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:40.705853282Z 2024-06-11 19:11:40,705 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:40.705862824Z 2024-06-11 19:11:40,705 - root - INFO - splitting the entity +2024-06-11T19:11:40.705864032Z 2024-06-11 19:11:40,705 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:40.706115074Z 2024-06-11 19:11:40,705 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/administration/integrations +2024-06-11T19:11:40.706119949Z 2024-06-11 19:11:40,705 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:11:41.271663991Z 2024-06-11 19:11:41,271 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:11:41.608486783Z 127.0.0.1 - - [11/Jun/2024 19:11:41] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:11:41.608732949Z 2024-06-11 19:11:41,608 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:11:43.814637909Z 2024-06-11 19:11:43,814 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.10 seconds +2024-06-11T19:11:43.821806700Z 2024-06-11 19:11:43,821 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133102.752966783,0): "sql txn" meta={id=00fe472c key=/Table/121/1/976691867862827009/0 iso=Serializable pri=0.03228406 epo=0 ts=1718133103.263349617,1 min=1718133100.709868866,0 seq=1} lock=true stat=PENDING rts=1718133100.709868866,0 wto=false gul=1718133101.209868866,0 +2024-06-11T19:11:43.821818909Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:43.821820534Z +2024-06-11T19:11:43.821821784Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:43.821823034Z 2024-06-11 19:11:43,821 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:43.830194950Z 2024-06-11 19:11:43,829 - __main__ - INFO - 1 +2024-06-11T19:11:43.830972450Z 2024-06-11 19:11:43,830 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:43.830979742Z 2024-06-11 19:11:43,830 - __main__ - INFO - ⏰⏰ total elapsed time: 3.65 seconds +2024-06-11T19:11:43.830981450Z 2024-06-11 19:11:43,830 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.845.845.1718132806324427975.163', data=b'\n7https://help.collaboard.app/administration/integrations(\x81\x80\xf0\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:43.831329742Z 2024-06-11 19:11:43,831 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.847.847.1718132806327801767.161', data=b'\n@https://help.collaboard.app/de/branding-gestalte-dein-collaboard(\x81\x80\xf4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:43.831383075Z 2024-06-11 19:11:43,831 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:43.831386242Z 2024-06-11 19:11:43,831 - __main__ - INFO - message: +2024-06-11T19:11:43.831387659Z url: "https://help.collaboard.app/de/branding-gestalte-dein-collaboard" +2024-06-11T19:11:43.831388950Z document_id: 976690891546492929 +2024-06-11T19:11:43.831390367Z connector_id: 976341399530373121 +2024-06-11T19:11:43.831391492Z file_type: URL +2024-06-11T19:11:43.831398450Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:43.831399659Z +2024-06-11T19:11:43.831400575Z 2024-06-11 19:11:43,831 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:43.831401617Z 2024-06-11 19:11:43,831 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:43.869064117Z 2024-06-11 19:11:43,868 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:43.869079409Z 2024-06-11 19:11:43,868 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/branding-gestalte-dein-collaboard +2024-06-11T19:11:43.869080992Z 2024-06-11 19:11:43,868 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/branding-gestalte-dein-collaboard +2024-06-11T19:11:44.007604867Z 2024-06-11 19:11:44,007 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.14 seconds +2024-06-11T19:11:44.021074576Z 2024-06-11 19:11:44,020 - Milvus_DB - INFO - deleting all entities related to document 976690891546492929 +2024-06-11T19:11:44.029990659Z 2024-06-11 19:11:44,029 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:44.070744617Z 2024-06-11 19:11:44,069 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:44.070760576Z 2024-06-11 19:11:44,070 - root - INFO - splitting the entity +2024-06-11T19:11:44.070944451Z 2024-06-11 19:11:44,070 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:44.071743826Z 2024-06-11 19:11:44,071 - URLSemantic - INFO - created 14 chunks for https://help.collaboard.app/de/branding-gestalte-dein-collaboard +2024-06-11T19:11:44.071809409Z 2024-06-11 19:11:44,071 - root - INFO - saving in milvus 14 chunk(s) +2024-06-11T19:11:44.679009909Z 2024-06-11 19:11:44,678 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:11:47.750970827Z 2024-06-11 19:11:47,750 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:11:47.756310994Z 2024-06-11 19:11:47,756 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133107.163410494,0): "sql txn" meta={id=7f25bed0 key=/Table/121/1/976691880761229313/0 iso=Serializable pri=0.05461178 epo=0 ts=1718133107.397785410,1 min=1718133104.095244742,0 seq=1} lock=true stat=PENDING rts=1718133104.095244742,0 wto=false gul=1718133104.595244742,0 +2024-06-11T19:11:47.756321286Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:11:47.756322744Z +2024-06-11T19:11:47.756323827Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:11:47.756324869Z 2024-06-11 19:11:47,756 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:11:47.760578286Z 2024-06-11 19:11:47,760 - __main__ - INFO - 1 +2024-06-11T19:11:47.761996911Z 2024-06-11 19:11:47,761 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:11:47.762001744Z 2024-06-11 19:11:47,761 - __main__ - INFO - ⏰⏰ total elapsed time: 3.93 seconds +2024-06-11T19:11:47.762003161Z 2024-06-11 19:11:47,761 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.847.847.1718132806327801767.161', data=b'\n@https://help.collaboard.app/de/branding-gestalte-dein-collaboard(\x81\x80\xf4\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:11:47.762092827Z 2024-06-11 19:11:47,761 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.849.849.1718132806330774142.159', data=b'\n.https://help.collaboard.app/de/konfigurationen(\x81\x80\xf8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:11:47.762096577Z 2024-06-11 19:11:47,761 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:11:47.762097702Z 2024-06-11 19:11:47,761 - __main__ - INFO - message: +2024-06-11T19:11:47.762098702Z url: "https://help.collaboard.app/de/konfigurationen" +2024-06-11T19:11:47.762099744Z document_id: 976690891546558465 +2024-06-11T19:11:47.762100744Z connector_id: 976341399530373121 +2024-06-11T19:11:47.762101702Z file_type: URL +2024-06-11T19:11:47.762102619Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:11:47.762103619Z +2024-06-11T19:11:47.762104494Z 2024-06-11 19:11:47,761 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:11:47.762105577Z 2024-06-11 19:11:47,761 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:11:47.795920786Z 2024-06-11 19:11:47,795 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:11:47.795935119Z 2024-06-11 19:11:47,795 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/konfigurationen +2024-06-11T19:11:47.795936494Z 2024-06-11 19:11:47,795 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/konfigurationen +2024-06-11T19:11:48.016648786Z 2024-06-11 19:11:48,016 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.22 seconds +2024-06-11T19:11:48.039451994Z 2024-06-11 19:11:48,039 - Milvus_DB - INFO - deleting all entities related to document 976690891546558465 +2024-06-11T19:11:48.049070744Z 2024-06-11 19:11:48,049 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:11:48.071743786Z 2024-06-11 19:11:48,071 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:11:48.071759911Z 2024-06-11 19:11:48,071 - root - INFO - splitting the entity +2024-06-11T19:11:48.071853744Z 2024-06-11 19:11:48,071 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:11:48.072004161Z 2024-06-11 19:11:48,071 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/konfigurationen +2024-06-11T19:11:48.072006327Z 2024-06-11 19:11:48,071 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:11:48.650965994Z 2024-06-11 19:11:48,650 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:11:51.199928787Z 2024-06-11 19:11:51,199 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.10 seconds +2024-06-11T19:11:51.796771496Z 2024-06-11 19:11:51,796 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:11:54.887022581Z 2024-06-11 19:11:54,886 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:11:55.453604123Z 2024-06-11 19:11:55,453 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:11:58.533959582Z 2024-06-11 19:11:58,532 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:11:59.155235833Z 2024-06-11 19:11:59,155 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:12:02.214388751Z 2024-06-11 19:12:02,214 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:12:02.784929668Z 2024-06-11 19:12:02,784 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:12:06.339386003Z 2024-06-11 19:12:06,338 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.12 seconds +2024-06-11T19:12:08.938117421Z 2024-06-11 19:12:08,937 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 2.04 seconds +2024-06-11T19:12:11.972451714Z 2024-06-11 19:12:11,972 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 5.61 seconds +2024-06-11T19:12:12.627185756Z 2024-06-11 19:12:12,626 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.12 seconds +2024-06-11T19:12:16.203177966Z 2024-06-11 19:12:16,202 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.22 seconds +2024-06-11T19:12:16.786836675Z 2024-06-11 19:12:16,786 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:12:19.899117593Z 2024-06-11 19:12:19,898 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.69 seconds +2024-06-11T19:12:20.502792426Z 2024-06-11 19:12:20,502 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:12:23.564834844Z 2024-06-11 19:12:23,564 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:12:24.308748553Z 2024-06-11 19:12:24,308 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:12:27.361772430Z 2024-06-11 19:12:27,361 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:12:28.051636055Z 2024-06-11 19:12:28,051 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:12:31.093744473Z 2024-06-11 19:12:31,093 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:12:31.101467473Z 2024-06-11 19:12:31,101 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133150.595304014,0): "sql txn" meta={id=a8df11b0 key=/Table/121/1/976692022788489217/0 iso=Serializable pri=0.03399462 epo=0 ts=1718133150.804701765,1 min=1718133147.385378096,0 seq=1} lock=true stat=PENDING rts=1718133147.385378096,0 wto=false gul=1718133147.885378096,0 +2024-06-11T19:12:31.101479056Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:12:31.101481848Z +2024-06-11T19:12:31.101483473Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:12:31.101485056Z 2024-06-11 19:12:31,101 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:12:31.108461181Z 2024-06-11 19:12:31,107 - __main__ - INFO - 1 +2024-06-11T19:12:31.110519181Z 2024-06-11 19:12:31,109 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:12:31.110526056Z 2024-06-11 19:12:31,109 - __main__ - INFO - ⏰⏰ total elapsed time: 43.35 seconds +2024-06-11T19:12:31.110528181Z 2024-06-11 19:12:31,109 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.849.849.1718132806330774142.159', data=b'\n.https://help.collaboard.app/de/konfigurationen(\x81\x80\xf8\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:12:31.111009348Z 2024-06-11 19:12:31,110 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.852.852.1718132806335699642.156', data=b'\n3https://help.collaboard.app/rocket.chat-integration(\x81\x80\xfe\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:12:31.111018431Z 2024-06-11 19:12:31,110 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:12:31.111020681Z 2024-06-11 19:12:31,110 - __main__ - INFO - message: +2024-06-11T19:12:31.111022140Z url: "https://help.collaboard.app/rocket.chat-integration" +2024-06-11T19:12:31.111023681Z document_id: 976690891546656769 +2024-06-11T19:12:31.111024931Z connector_id: 976341399530373121 +2024-06-11T19:12:31.111026223Z file_type: URL +2024-06-11T19:12:31.111027681Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:12:31.111029098Z +2024-06-11T19:12:31.111030265Z 2024-06-11 19:12:31,110 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:12:31.111046598Z 2024-06-11 19:12:31,110 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:12:31.146839098Z 2024-06-11 19:12:31,146 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:12:31.146847265Z 2024-06-11 19:12:31,146 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/rocket.chat-integration +2024-06-11T19:12:31.146849098Z 2024-06-11 19:12:31,146 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/rocket.chat-integration +2024-06-11T19:12:31.352386181Z 2024-06-11 19:12:31,352 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.21 seconds +2024-06-11T19:12:31.369873890Z 2024-06-11 19:12:31,369 - Milvus_DB - INFO - deleting all entities related to document 976690891546656769 +2024-06-11T19:12:31.418115473Z 2024-06-11 19:12:31,417 - Milvus_DB - INFO - ⏰ total elapsed time: 0.05 seconds +2024-06-11T19:12:31.436140223Z 2024-06-11 19:12:31,436 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:12:31.436146598Z 2024-06-11 19:12:31,436 - root - INFO - splitting the entity +2024-06-11T19:12:31.436147931Z 2024-06-11 19:12:31,436 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:12:31.436871181Z 2024-06-11 19:12:31,436 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/rocket.chat-integration +2024-06-11T19:12:31.436880848Z 2024-06-11 19:12:31,436 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:12:32.005601890Z 2024-06-11 19:12:32,005 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:12:35.053938433Z 2024-06-11 19:12:35,053 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.60 seconds +2024-06-11T19:12:35.066746475Z 2024-06-11 19:12:35,066 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718133154.579306975,0): "sql txn" meta={id=09684bbb key=/Table/121/1/976692035771334657/0 iso=Serializable pri=0.05604685 epo=0 ts=1718133154.749762058,1 min=1718133151.445662890,0 seq=1} lock=true stat=PENDING rts=1718133151.445662890,0 wto=false gul=1718133151.945662890,0 +2024-06-11T19:12:35.066763350Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:12:35.066767517Z +2024-06-11T19:12:35.066771267Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:12:35.066774642Z 2024-06-11 19:12:35,066 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:12:35.077388475Z 2024-06-11 19:12:35,076 - __main__ - INFO - 1 +2024-06-11T19:12:35.080202683Z 2024-06-11 19:12:35,079 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:12:35.080214350Z 2024-06-11 19:12:35,079 - __main__ - INFO - ⏰⏰ total elapsed time: 3.97 seconds +2024-06-11T19:12:35.080226808Z 2024-06-11 19:12:35,079 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.852.852.1718132806335699642.156', data=b'\n3https://help.collaboard.app/rocket.chat-integration(\x81\x80\xfe\xef\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:12:35.082498767Z 2024-06-11 19:12:35,081 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.854.854.1718132806338091975.154', data=b'\n:https://help.collaboard.app/customize-terms-of-service-tos(\x81\x80\x82\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:12:35.082509308Z 2024-06-11 19:12:35,081 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:12:35.082512100Z 2024-06-11 19:12:35,081 - __main__ - INFO - message: +2024-06-11T19:12:35.082514017Z url: "https://help.collaboard.app/customize-terms-of-service-tos" +2024-06-11T19:12:35.082515975Z document_id: 976690891546722305 +2024-06-11T19:12:35.082518683Z connector_id: 976341399530373121 +2024-06-11T19:12:35.082520225Z file_type: URL +2024-06-11T19:12:35.082521767Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:12:35.082523517Z +2024-06-11T19:12:35.082525058Z 2024-06-11 19:12:35,081 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:12:35.082526808Z 2024-06-11 19:12:35,081 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:12:35.121041850Z 2024-06-11 19:12:35,120 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:12:35.121053600Z 2024-06-11 19:12:35,120 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/customize-terms-of-service-tos +2024-06-11T19:12:35.121055100Z 2024-06-11 19:12:35,120 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/customize-terms-of-service-tos +2024-06-11T19:12:35.515382350Z 2024-06-11 19:12:35,515 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.39 seconds +2024-06-11T19:12:35.527716267Z 2024-06-11 19:12:35,527 - Milvus_DB - INFO - deleting all entities related to document 976690891546722305 +2024-06-11T19:12:35.536306100Z 2024-06-11 19:12:35,536 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:12:35.563647308Z 2024-06-11 19:12:35,563 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:12:35.563661058Z 2024-06-11 19:12:35,563 - root - INFO - splitting the entity +2024-06-11T19:12:35.563662433Z 2024-06-11 19:12:35,563 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:12:35.563895558Z 2024-06-11 19:12:35,563 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/customize-terms-of-service-tos +2024-06-11T19:12:35.563905058Z 2024-06-11 19:12:35,563 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:12:36.149838100Z 2024-06-11 19:12:36,149 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:12:39.209478394Z 2024-06-11 19:12:39,206 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:12:39.839060935Z 2024-06-11 19:12:39,838 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:12:41.699702145Z 127.0.0.1 - - [11/Jun/2024 19:12:41] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:12:41.699951020Z 2024-06-11 19:12:41,699 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:12:42.879753770Z 2024-06-11 19:12:42,879 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:12:43.497127312Z 2024-06-11 19:12:43,497 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:12:46.556783314Z 2024-06-11 19:12:46,556 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:12:47.128894064Z 2024-06-11 19:12:47,128 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.03 seconds +2024-06-11T19:12:50.213557815Z 2024-06-11 19:12:50,213 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:12:50.809165482Z 2024-06-11 19:12:50,809 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:25:13.186462765Z 2024-06-11 19:25:13,186 - __main__ - INFO - Logging configured with level INFO and format %(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s +2024-06-11T19:25:13.186767098Z 2024-06-11 19:25:13,186 - ReadinessProbe - INFO - Initializing readiness probe server +2024-06-11T19:25:13.186934098Z 2024-06-11 19:25:13,186 - __main__ - INFO - 🛠️ service starting.. +2024-06-11T19:25:13.187432890Z 2024-06-11 19:25:13,187 - ReadinessProbe - INFO - Readiness probe server started on ('', 8080)8080 +2024-06-11T19:25:13.187540182Z 2024-06-11 19:25:13,187 - JetStreamEventSubscriber - INFO - 🔌 connecting to nats endpoint nats://nats:4222 .. +2024-06-11T19:25:13.191364557Z 2024-06-11 19:25:13,191 - JetStreamEventSubscriber - INFO - successfully connected nats://nats:4222 +2024-06-11T19:25:13.192634307Z 2024-06-11 19:25:13,192 - JetStreamEventSubscriber - INFO - subscribing to jetstream semantic - semantic_activity .. +2024-06-11T19:25:13.193130890Z 2024-06-11 19:25:13,193 - JetStreamEventSubscriber - INFO - successfully subscribed to jetstream semantic - semantic_activity +2024-06-11T19:25:13.199448265Z 2024-06-11 19:25:13,199 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.861.861.1718132806347643392.147', data=b'\nNhttps://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben(\x81\x80\x90\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:25:13.199463973Z 2024-06-11 19:25:13,199 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:25:13.199465973Z 2024-06-11 19:25:13,199 - __main__ - INFO - message: +2024-06-11T19:25:13.199467057Z url: "https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben" +2024-06-11T19:25:13.199468265Z document_id: 976690891546951681 +2024-06-11T19:25:13.199469265Z connector_id: 976341399530373121 +2024-06-11T19:25:13.199470182Z file_type: URL +2024-06-11T19:25:13.199471057Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:25:13.199472098Z +2024-06-11T19:25:13.199472932Z 2024-06-11 19:25:13,199 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:25:13.199473973Z 2024-06-11 19:25:13,199 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:25:13.266366598Z 2024-06-11 19:25:13,265 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:25:13.266389890Z 2024-06-11 19:25:13,265 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben +2024-06-11T19:25:13.266391723Z 2024-06-11 19:25:13,266 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben +2024-06-11T19:25:13.380011682Z 2024-06-11 19:25:13,379 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:25:13.390828973Z 2024-06-11 19:25:13,390 - Milvus_DB - INFO - deleting all entities related to document 976690891546951681 +2024-06-11T19:25:13.413046557Z 2024-06-11 19:25:13,412 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:25:13.438185432Z 2024-06-11 19:25:13,438 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:25:13.438200390Z 2024-06-11 19:25:13,438 - root - INFO - splitting the entity +2024-06-11T19:25:13.438201890Z 2024-06-11 19:25:13,438 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:25:13.438586723Z 2024-06-11 19:25:13,438 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben +2024-06-11T19:25:13.438593348Z 2024-06-11 19:25:13,438 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:25:14.045852807Z 2024-06-11 19:25:14,045 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:25:17.113195100Z 2024-06-11 19:25:17,112 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:25:17.713420267Z 2024-06-11 19:25:17,713 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:25:21.283963602Z 2024-06-11 19:25:21,283 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.16 seconds +2024-06-11T19:25:21.952938477Z 2024-06-11 19:25:21,952 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:25:25.031657729Z 2024-06-11 19:25:25,031 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.72 seconds +2024-06-11T19:25:25.623896437Z 2024-06-11 19:25:25,623 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:25:28.198847564Z 2024-06-11 19:25:28,198 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.16 seconds +2024-06-11T19:25:28.830610147Z 2024-06-11 19:25:28,830 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:25:31.949924774Z 2024-06-11 19:25:31,949 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.74 seconds +2024-06-11T19:25:32.577539732Z 2024-06-11 19:25:32,577 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:25:35.635882359Z 2024-06-11 19:25:35,635 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:25:36.229672234Z 2024-06-11 19:25:36,229 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:25:38.789105444Z 2024-06-11 19:25:38,788 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.14 seconds +2024-06-11T19:25:39.420054111Z 2024-06-11 19:25:39,419 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:25:42.502260070Z 2024-06-11 19:25:42,501 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:25:43.113318821Z 2024-06-11 19:25:43,113 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:25:46.186020531Z 2024-06-11 19:25:46,185 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:25:46.816881073Z 2024-06-11 19:25:46,816 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:25:49.920959032Z 2024-06-11 19:25:49,920 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.72 seconds +2024-06-11T19:25:50.614690533Z 2024-06-11 19:25:50,614 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.11 seconds +2024-06-11T19:25:53.667383576Z 2024-06-11 19:25:53,666 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:25:53.677215367Z 2024-06-11 19:25:53,677 - URLSemantic - INFO - ⏰ total elapsed time: 40.41 seconds +2024-06-11T19:25:53.677225326Z 2024-06-11 19:25:53,677 - URLSemantic - INFO - 📖 number of docs analyzed: 11 +2024-06-11T19:25:53.682485076Z 2024-06-11 19:25:53,682 - __main__ - INFO - 1 +2024-06-11T19:25:53.683330992Z 2024-06-11 19:25:53,683 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 11 +2024-06-11T19:25:53.683339284Z 2024-06-11 19:25:53,683 - __main__ - INFO - ⏰⏰ total elapsed time: 40.48 seconds +2024-06-11T19:25:53.683341159Z 2024-06-11 19:25:53,683 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.861.861.1718132806347643392.147', data=b'\nNhttps://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben(\x81\x80\x90\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:25:53.684566867Z 2024-06-11 19:25:53,684 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.862.862.1718132806349408309.146', data=b'\n@https://help.collaboard.app/how-to-buy-a-collaboard-subscription(\x81\x80\x92\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:25:53.684572242Z 2024-06-11 19:25:53,684 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:25:53.684573617Z 2024-06-11 19:25:53,684 - __main__ - INFO - message: +2024-06-11T19:25:53.684574784Z url: "https://help.collaboard.app/how-to-buy-a-collaboard-subscription" +2024-06-11T19:25:53.684575909Z document_id: 976690891546984449 +2024-06-11T19:25:53.684576867Z connector_id: 976341399530373121 +2024-06-11T19:25:53.684577784Z file_type: URL +2024-06-11T19:25:53.684578701Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:25:53.684579867Z +2024-06-11T19:25:53.684580784Z 2024-06-11 19:25:53,684 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:25:53.684581826Z 2024-06-11 19:25:53,684 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:25:53.718749618Z 2024-06-11 19:25:53,718 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:25:53.718764576Z 2024-06-11 19:25:53,718 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/how-to-buy-a-collaboard-subscription +2024-06-11T19:25:53.718766784Z 2024-06-11 19:25:53,718 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/how-to-buy-a-collaboard-subscription +2024-06-11T19:25:53.946792243Z 2024-06-11 19:25:53,946 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.23 seconds +2024-06-11T19:25:53.965426743Z 2024-06-11 19:25:53,965 - Milvus_DB - INFO - deleting all entities related to document 976690891546984449 +2024-06-11T19:25:53.982681951Z 2024-06-11 19:25:53,982 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:25:53.993184159Z 2024-06-11 19:25:53,993 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:25:53.993195368Z 2024-06-11 19:25:53,993 - root - INFO - splitting the entity +2024-06-11T19:25:53.993197451Z 2024-06-11 19:25:53,993 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:25:53.993534201Z 2024-06-11 19:25:53,993 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/how-to-buy-a-collaboard-subscription +2024-06-11T19:25:53.993538659Z 2024-06-11 19:25:53,993 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:25:54.595751201Z 2024-06-11 19:25:54,595 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:25:57.633583578Z 2024-06-11 19:25:57,633 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:25:58.262224370Z 2024-06-11 19:25:58,262 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:26:01.384709038Z 2024-06-11 19:26:01,384 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.74 seconds +2024-06-11T19:26:02.034935913Z 2024-06-11 19:26:02,034 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:26:05.090684415Z 2024-06-11 19:26:05,090 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.69 seconds +2024-06-11T19:26:05.673789957Z 2024-06-11 19:26:05,673 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:26:08.247533666Z 2024-06-11 19:26:08,247 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.15 seconds +2024-06-11T19:26:08.881793458Z 2024-06-11 19:26:08,881 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:26:11.972979543Z 2024-06-11 19:26:11,972 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.72 seconds +2024-06-11T19:26:12.424893460Z 127.0.0.1 - - [11/Jun/2024 19:26:12] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:26:12.425439543Z 2024-06-11 19:26:12,424 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:26:12.575182418Z 2024-06-11 19:26:12,575 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:26:15.625084086Z 2024-06-11 19:26:15,624 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:26:16.223492128Z 2024-06-11 19:26:16,223 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:26:19.310934296Z 2024-06-11 19:26:19,310 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:26:19.937662838Z 2024-06-11 19:26:19,937 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:26:23.027083381Z 2024-06-11 19:26:23,026 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:26:23.732315090Z 2024-06-11 19:26:23,732 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.17 seconds +2024-06-11T19:26:26.819227467Z 2024-06-11 19:26:26,818 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.78 seconds +2024-06-11T19:26:27.448813717Z 2024-06-11 19:26:27,448 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:26:30.532952052Z 2024-06-11 19:26:30,532 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:26:31.167527469Z 2024-06-11 19:26:31,167 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:26:34.233867595Z 2024-06-11 19:26:34,233 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:26:34.248303970Z 2024-06-11 19:26:34,248 - URLSemantic - INFO - ⏰ total elapsed time: 40.53 seconds +2024-06-11T19:26:34.248320554Z 2024-06-11 19:26:34,248 - URLSemantic - INFO - 📖 number of docs analyzed: 11 +2024-06-11T19:26:34.254836012Z 2024-06-11 19:26:34,254 - __main__ - INFO - 1 +2024-06-11T19:26:34.256528637Z 2024-06-11 19:26:34,255 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 11 +2024-06-11T19:26:34.256538470Z 2024-06-11 19:26:34,255 - __main__ - INFO - ⏰⏰ total elapsed time: 40.57 seconds +2024-06-11T19:26:34.256540512Z 2024-06-11 19:26:34,255 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.862.862.1718132806349408309.146', data=b'\n@https://help.collaboard.app/how-to-buy-a-collaboard-subscription(\x81\x80\x92\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:26:34.264965845Z 2024-06-11 19:26:34,264 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.863.863.1718132806350725225.145', data=b'\nPhttps://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben-1(\x81\x80\x94\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:26:34.264971804Z 2024-06-11 19:26:34,264 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:26:34.264973387Z 2024-06-11 19:26:34,264 - __main__ - INFO - message: +2024-06-11T19:26:34.264974470Z url: "https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben-1" +2024-06-11T19:26:34.264975762Z document_id: 976690891547017217 +2024-06-11T19:26:34.264976845Z connector_id: 976341399530373121 +2024-06-11T19:26:34.264977845Z file_type: URL +2024-06-11T19:26:34.264978804Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:26:34.264985887Z +2024-06-11T19:26:34.264987095Z 2024-06-11 19:26:34,264 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:26:34.264988220Z 2024-06-11 19:26:34,264 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:26:34.301986804Z 2024-06-11 19:26:34,301 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:26:34.302005220Z 2024-06-11 19:26:34,301 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben-1 +2024-06-11T19:26:34.302007345Z 2024-06-11 19:26:34,301 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben-1 +2024-06-11T19:26:34.445298762Z 2024-06-11 19:26:34,445 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.14 seconds +2024-06-11T19:26:34.460251220Z 2024-06-11 19:26:34,460 - Milvus_DB - INFO - deleting all entities related to document 976690891547017217 +2024-06-11T19:26:34.473550012Z 2024-06-11 19:26:34,473 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:26:34.486050387Z 2024-06-11 19:26:34,485 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:26:34.486064554Z 2024-06-11 19:26:34,485 - root - INFO - splitting the entity +2024-06-11T19:26:34.486068720Z 2024-06-11 19:26:34,485 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:26:34.486560929Z 2024-06-11 19:26:34,486 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben-1 +2024-06-11T19:26:34.486566304Z 2024-06-11 19:26:34,486 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:26:35.069913054Z 2024-06-11 19:26:35,069 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:26:38.107784805Z 2024-06-11 19:26:38,107 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:26:38.745431097Z 2024-06-11 19:26:38,745 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:26:41.795672265Z 2024-06-11 19:26:41,795 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:26:42.460517016Z 2024-06-11 19:26:42,460 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.14 seconds +2024-06-11T19:26:45.499738142Z 2024-06-11 19:26:45,499 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:26:46.088932392Z 2024-06-11 19:26:46,088 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:26:49.174435852Z 2024-06-11 19:26:49,173 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:26:49.823090936Z 2024-06-11 19:26:49,823 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:26:52.926047396Z 2024-06-11 19:26:52,925 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.73 seconds +2024-06-11T19:26:53.565365354Z 2024-06-11 19:26:53,565 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:26:56.614744814Z 2024-06-11 19:26:56,614 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:26:57.216639564Z 2024-06-11 19:26:57,216 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:27:00.298672774Z 2024-06-11 19:27:00,298 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:27:00.930602816Z 2024-06-11 19:27:00,930 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:27:04.007111359Z 2024-06-11 19:27:04,006 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:27:04.633755526Z 2024-06-11 19:27:04,633 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:27:07.236085736Z 2024-06-11 19:27:07,235 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.22 seconds +2024-06-11T19:27:07.902428695Z 2024-06-11 19:27:07,902 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.14 seconds +2024-06-11T19:27:10.958291738Z 2024-06-11 19:27:10,957 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:27:11.588914238Z 2024-06-11 19:27:11,588 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:27:12.521536072Z 127.0.0.1 - - [11/Jun/2024 19:27:12] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:27:12.521551363Z 2024-06-11 19:27:12,521 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:27:14.640119906Z 2024-06-11 19:27:14,639 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:27:14.649684031Z 2024-06-11 19:27:14,649 - URLSemantic - INFO - ⏰ total elapsed time: 40.35 seconds +2024-06-11T19:27:14.649691573Z 2024-06-11 19:27:14,649 - URLSemantic - INFO - 📖 number of docs analyzed: 11 +2024-06-11T19:27:14.654630614Z 2024-06-11 19:27:14,654 - __main__ - INFO - 1 +2024-06-11T19:27:14.656123948Z 2024-06-11 19:27:14,655 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 11 +2024-06-11T19:27:14.656129739Z 2024-06-11 19:27:14,655 - __main__ - INFO - ⏰⏰ total elapsed time: 40.39 seconds +2024-06-11T19:27:14.656131406Z 2024-06-11 19:27:14,655 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.863.863.1718132806350725225.145', data=b'\nPhttps://help.collaboard.app/de/wie-kann-ich-ein-collaboard-abonnement-erwerben-1(\x81\x80\x94\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:27:14.663357948Z 2024-06-11 19:27:14,663 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.864.864.1718132806352075850.144', data=b'\nFhttps://help.collaboard.app/how-to-redeem-a-license-code-in-collaboard(\x81\x80\x96\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:27:14.663366739Z 2024-06-11 19:27:14,663 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:27:14.663842573Z 2024-06-11 19:27:14,663 - __main__ - INFO - message: +2024-06-11T19:27:14.663851281Z url: "https://help.collaboard.app/how-to-redeem-a-license-code-in-collaboard" +2024-06-11T19:27:14.663852989Z document_id: 976690891547049985 +2024-06-11T19:27:14.663856406Z connector_id: 976341399530373121 +2024-06-11T19:27:14.663857614Z file_type: URL +2024-06-11T19:27:14.663864531Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:27:14.663865698Z +2024-06-11T19:27:14.663866614Z 2024-06-11 19:27:14,663 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:27:14.663868031Z 2024-06-11 19:27:14,663 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:27:14.700726781Z 2024-06-11 19:27:14,700 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:27:14.700740573Z 2024-06-11 19:27:14,700 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/how-to-redeem-a-license-code-in-collaboard +2024-06-11T19:27:14.700742239Z 2024-06-11 19:27:14,700 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/how-to-redeem-a-license-code-in-collaboard +2024-06-11T19:27:14.790819948Z 2024-06-11 19:27:14,790 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.09 seconds +2024-06-11T19:27:14.802596990Z 2024-06-11 19:27:14,802 - Milvus_DB - INFO - deleting all entities related to document 976690891547049985 +2024-06-11T19:27:14.812965365Z 2024-06-11 19:27:14,812 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:27:14.822980531Z 2024-06-11 19:27:14,822 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:27:14.822989198Z 2024-06-11 19:27:14,822 - root - INFO - splitting the entity +2024-06-11T19:27:14.822990573Z 2024-06-11 19:27:14,822 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:27:14.823306781Z 2024-06-11 19:27:14,823 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/how-to-redeem-a-license-code-in-collaboard +2024-06-11T19:27:14.823311115Z 2024-06-11 19:27:14,823 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:27:15.421889281Z 2024-06-11 19:27:15,421 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:27:18.467842950Z 2024-06-11 19:27:18,467 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:27:19.126039825Z 2024-06-11 19:27:19,125 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:27:22.237238243Z 2024-06-11 19:27:22,236 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.75 seconds +2024-06-11T19:27:22.981210660Z 2024-06-11 19:27:22,981 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.18 seconds +2024-06-11T19:27:26.028302203Z 2024-06-11 19:27:26,027 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.77 seconds +2024-06-11T19:27:26.642444370Z 2024-06-11 19:27:26,642 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:27:29.199006288Z 2024-06-11 19:27:29,198 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.16 seconds +2024-06-11T19:27:29.820389122Z 2024-06-11 19:27:29,820 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:27:32.892080581Z 2024-06-11 19:27:32,891 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:27:33.710603582Z 2024-06-11 19:27:33,710 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.29 seconds +2024-06-11T19:27:37.259203959Z 2024-06-11 19:27:37,258 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.36 seconds +2024-06-11T19:27:37.963402542Z 2024-06-11 19:27:37,963 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.15 seconds +2024-06-11T19:27:41.006547210Z 2024-06-11 19:27:41,006 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.73 seconds +2024-06-11T19:27:41.630036336Z 2024-06-11 19:27:41,629 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:27:44.777123045Z 2024-06-11 19:27:44,776 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.76 seconds +2024-06-11T19:27:44.792805420Z 2024-06-11 19:27:44,792 - URLSemantic - INFO - ⏰ total elapsed time: 30.09 seconds +2024-06-11T19:27:44.792821254Z 2024-06-11 19:27:44,792 - URLSemantic - INFO - 📖 number of docs analyzed: 8 +2024-06-11T19:27:44.799855420Z 2024-06-11 19:27:44,799 - __main__ - INFO - 1 +2024-06-11T19:27:44.802003420Z 2024-06-11 19:27:44,801 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 8 +2024-06-11T19:27:44.802011420Z 2024-06-11 19:27:44,801 - __main__ - INFO - ⏰⏰ total elapsed time: 30.14 seconds +2024-06-11T19:27:44.802012962Z 2024-06-11 19:27:44,801 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.864.864.1718132806352075850.144', data=b'\nFhttps://help.collaboard.app/how-to-redeem-a-license-code-in-collaboard(\x81\x80\x96\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:27:44.810999629Z 2024-06-11 19:27:44,810 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.865.865.1718132806353263684.143', data=b'\nOhttps://help.collaboard.app/de/wie-l\xc3\xb6se-ich-einen-lizenzcode-in-collaboard-ein(\x81\x80\x98\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:27:44.811012379Z 2024-06-11 19:27:44,810 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:27:44.811015670Z 2024-06-11 19:27:44,810 - __main__ - INFO - message: +2024-06-11T19:27:44.811017254Z url: "https://help.collaboard.app/de/wie-löse-ich-einen-lizenzcode-in-collaboard-ein" +2024-06-11T19:27:44.811018462Z document_id: 976690891547082753 +2024-06-11T19:27:44.811019420Z connector_id: 976341399530373121 +2024-06-11T19:27:44.811020379Z file_type: URL +2024-06-11T19:27:44.811021587Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:27:44.811022754Z +2024-06-11T19:27:44.811052587Z 2024-06-11 19:27:44,810 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:27:44.811175837Z 2024-06-11 19:27:44,810 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:27:44.871705129Z 2024-06-11 19:27:44,871 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:27:44.871731546Z 2024-06-11 19:27:44,871 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/wie-löse-ich-einen-lizenzcode-in-collaboard-ein +2024-06-11T19:27:44.871735546Z 2024-06-11 19:27:44,871 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/wie-löse-ich-einen-lizenzcode-in-collaboard-ein +2024-06-11T19:27:45.054396629Z 2024-06-11 19:27:45,053 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.18 seconds +2024-06-11T19:27:45.073699629Z 2024-06-11 19:27:45,073 - Milvus_DB - INFO - deleting all entities related to document 976690891547082753 +2024-06-11T19:27:45.089453546Z 2024-06-11 19:27:45,089 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:27:45.103041962Z 2024-06-11 19:27:45,102 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:27:45.103055587Z 2024-06-11 19:27:45,102 - root - INFO - splitting the entity +2024-06-11T19:27:45.103057754Z 2024-06-11 19:27:45,102 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:27:45.103338546Z 2024-06-11 19:27:45,103 - URLSemantic - INFO - created 8 chunks for https://help.collaboard.app/de/wie-löse-ich-einen-lizenzcode-in-collaboard-ein +2024-06-11T19:27:45.103343296Z 2024-06-11 19:27:45,103 - root - INFO - saving in milvus 8 chunk(s) +2024-06-11T19:27:45.697800296Z 2024-06-11 19:27:45,697 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:27:49.262539464Z 2024-06-11 19:27:49,262 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.15 seconds +2024-06-11T19:27:49.931789340Z 2024-06-11 19:27:49,931 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:27:53.022205299Z 2024-06-11 19:27:53,021 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.74 seconds +2024-06-11T19:27:53.825410925Z 2024-06-11 19:27:53,825 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.26 seconds +2024-06-11T19:27:56.892504343Z 2024-06-11 19:27:56,892 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.86 seconds +2024-06-11T19:27:57.542355135Z 2024-06-11 19:27:57,542 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.11 seconds +2024-06-11T19:28:00.592758136Z 2024-06-11 19:28:00,592 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.69 seconds +2024-06-11T19:28:01.247125928Z 2024-06-11 19:28:01,247 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.11 seconds +2024-06-11T19:28:03.798469055Z 2024-06-11 19:28:03,798 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.20 seconds +2024-06-11T19:28:04.425006097Z 2024-06-11 19:28:04,424 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:28:07.509804015Z 2024-06-11 19:28:07,509 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:28:08.166827473Z 2024-06-11 19:28:08,166 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:28:11.240211808Z 2024-06-11 19:28:11,239 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:28:11.863196975Z 2024-06-11 19:28:11,863 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:28:12.640588475Z 127.0.0.1 - - [11/Jun/2024 19:28:12] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:28:12.640910725Z 2024-06-11 19:28:12,640 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:28:14.931765310Z 2024-06-11 19:28:14,931 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:28:14.939562102Z 2024-06-11 19:28:14,939 - URLSemantic - INFO - ⏰ total elapsed time: 30.07 seconds +2024-06-11T19:28:14.939573143Z 2024-06-11 19:28:14,939 - URLSemantic - INFO - 📖 number of docs analyzed: 8 +2024-06-11T19:28:14.944359310Z 2024-06-11 19:28:14,944 - __main__ - INFO - 1 +2024-06-11T19:28:14.945817893Z 2024-06-11 19:28:14,945 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 8 +2024-06-11T19:28:14.945824185Z 2024-06-11 19:28:14,945 - __main__ - INFO - ⏰⏰ total elapsed time: 30.13 seconds +2024-06-11T19:28:14.945825727Z 2024-06-11 19:28:14,945 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.865.865.1718132806353263684.143', data=b'\nOhttps://help.collaboard.app/de/wie-l\xc3\xb6se-ich-einen-lizenzcode-in-collaboard-ein(\x81\x80\x98\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:28:14.952441727Z 2024-06-11 19:28:14,952 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.866.866.1718132806354701225.142', data=b'\n>https://help.collaboard.app/what-is-a-guest-user-in-collaboard(\x81\x80\x9a\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:28:14.952447768Z 2024-06-11 19:28:14,952 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:28:14.952449227Z 2024-06-11 19:28:14,952 - __main__ - INFO - message: +2024-06-11T19:28:14.952450268Z url: "https://help.collaboard.app/what-is-a-guest-user-in-collaboard" +2024-06-11T19:28:14.952451435Z document_id: 976690891547115521 +2024-06-11T19:28:14.952452393Z connector_id: 976341399530373121 +2024-06-11T19:28:14.952453393Z file_type: URL +2024-06-11T19:28:14.952454310Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:28:14.952463018Z +2024-06-11T19:28:14.952559102Z 2024-06-11 19:28:14,952 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:28:14.952561643Z 2024-06-11 19:28:14,952 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:28:14.998426352Z 2024-06-11 19:28:14,997 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:28:14.998447352Z 2024-06-11 19:28:14,998 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/what-is-a-guest-user-in-collaboard +2024-06-11T19:28:14.998449727Z 2024-06-11 19:28:14,998 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/what-is-a-guest-user-in-collaboard +2024-06-11T19:28:15.157196602Z 2024-06-11 19:28:15,156 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.16 seconds +2024-06-11T19:28:15.171114977Z 2024-06-11 19:28:15,170 - Milvus_DB - INFO - deleting all entities related to document 976690891547115521 +2024-06-11T19:28:15.183652685Z 2024-06-11 19:28:15,183 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:28:15.194609268Z 2024-06-11 19:28:15,194 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:28:15.194623060Z 2024-06-11 19:28:15,194 - root - INFO - splitting the entity +2024-06-11T19:28:15.194624560Z 2024-06-11 19:28:15,194 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:28:15.194765393Z 2024-06-11 19:28:15,194 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/what-is-a-guest-user-in-collaboard +2024-06-11T19:28:15.194767643Z 2024-06-11 19:28:15,194 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:28:15.777541560Z 2024-06-11 19:28:15,777 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:28:18.845146645Z 2024-06-11 19:28:18,844 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:28:19.476208645Z 2024-06-11 19:28:19,476 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:28:22.551601730Z 2024-06-11 19:28:22,550 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:28:23.204296647Z 2024-06-11 19:28:23,204 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.12 seconds +2024-06-11T19:28:26.248057440Z 2024-06-11 19:28:26,247 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.68 seconds +2024-06-11T19:28:26.853751607Z 2024-06-11 19:28:26,853 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:28:29.906493484Z 2024-06-11 19:28:29,906 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:28:30.532269359Z 2024-06-11 19:28:30,532 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:28:33.613699735Z 2024-06-11 19:28:33,612 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:28:34.272362027Z 2024-06-11 19:28:34,272 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:28:36.830363279Z 2024-06-11 19:28:36,829 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.19 seconds +2024-06-11T19:28:37.416883196Z 2024-06-11 19:28:37,416 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:28:40.504922697Z 2024-06-11 19:28:40,504 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:28:41.106499739Z 2024-06-11 19:28:41,105 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:28:44.200982907Z 2024-06-11 19:28:44,200 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.69 seconds +2024-06-11T19:28:44.798734991Z 2024-06-11 19:28:44,798 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:28:47.900231201Z 2024-06-11 19:28:47,900 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.69 seconds +2024-06-11T19:28:47.909876409Z 2024-06-11 19:28:47,909 - URLSemantic - INFO - ⏰ total elapsed time: 32.91 seconds +2024-06-11T19:28:47.909883826Z 2024-06-11 19:28:47,909 - URLSemantic - INFO - 📖 number of docs analyzed: 9 +2024-06-11T19:28:47.914543617Z 2024-06-11 19:28:47,914 - __main__ - INFO - 1 +2024-06-11T19:28:47.916578492Z 2024-06-11 19:28:47,915 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 9 +2024-06-11T19:28:47.916586576Z 2024-06-11 19:28:47,916 - __main__ - INFO - ⏰⏰ total elapsed time: 32.96 seconds +2024-06-11T19:28:47.916588242Z 2024-06-11 19:28:47,916 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.866.866.1718132806354701225.142', data=b'\n>https://help.collaboard.app/what-is-a-guest-user-in-collaboard(\x81\x80\x9a\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:28:47.924585117Z 2024-06-11 19:28:47,923 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.867.867.1718132806355894184.141', data=b'\n+https://help.collaboard.app/de/gastbenutzer(\x81\x80\x9c\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:28:47.924591909Z 2024-06-11 19:28:47,923 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:28:47.924594326Z 2024-06-11 19:28:47,923 - __main__ - INFO - message: +2024-06-11T19:28:47.924595951Z url: "https://help.collaboard.app/de/gastbenutzer" +2024-06-11T19:28:47.924597159Z document_id: 976690891547148289 +2024-06-11T19:28:47.924598201Z connector_id: 976341399530373121 +2024-06-11T19:28:47.924599242Z file_type: URL +2024-06-11T19:28:47.924607992Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:28:47.924609992Z +2024-06-11T19:28:47.924610909Z 2024-06-11 19:28:47,923 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:28:47.924612076Z 2024-06-11 19:28:47,923 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:28:47.959929951Z 2024-06-11 19:28:47,959 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:28:47.959942992Z 2024-06-11 19:28:47,959 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/gastbenutzer +2024-06-11T19:28:47.959944951Z 2024-06-11 19:28:47,959 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/gastbenutzer +2024-06-11T19:28:48.167541992Z 2024-06-11 19:28:48,167 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.21 seconds +2024-06-11T19:28:48.184241784Z 2024-06-11 19:28:48,183 - Milvus_DB - INFO - deleting all entities related to document 976690891547148289 +2024-06-11T19:28:48.197317826Z 2024-06-11 19:28:48,197 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:28:48.208496701Z 2024-06-11 19:28:48,208 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:28:48.208507617Z 2024-06-11 19:28:48,208 - root - INFO - splitting the entity +2024-06-11T19:28:48.208509451Z 2024-06-11 19:28:48,208 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:28:48.209121617Z 2024-06-11 19:28:48,208 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/gastbenutzer +2024-06-11T19:28:48.209132117Z 2024-06-11 19:28:48,208 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:28:48.799416993Z 2024-06-11 19:28:48,799 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:28:51.876955952Z 2024-06-11 19:28:51,876 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:28:52.565909494Z 2024-06-11 19:28:52,565 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.16 seconds +2024-06-11T19:28:55.593509329Z 2024-06-11 19:28:55,593 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:28:56.227615288Z 2024-06-11 19:28:56,227 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:28:59.319095623Z 2024-06-11 19:28:59,318 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:28:59.915456623Z 2024-06-11 19:28:59,914 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:29:02.997891833Z 2024-06-11 19:29:02,997 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:29:03.660181958Z 2024-06-11 19:29:03,659 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.12 seconds +2024-06-11T19:29:06.214100959Z 2024-06-11 19:29:06,213 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.21 seconds +2024-06-11T19:29:06.848014418Z 2024-06-11 19:29:06,847 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:29:09.945985294Z 2024-06-11 19:29:09,945 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.72 seconds +2024-06-11T19:29:10.552343503Z 2024-06-11 19:29:10,552 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:29:12.793064462Z 127.0.0.1 - - [11/Jun/2024 19:29:12] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:29:12.793493587Z 2024-06-11 19:29:12,792 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:29:13.589089796Z 2024-06-11 19:29:13,588 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:29:14.252790463Z 2024-06-11 19:29:14,252 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:29:16.828374464Z 2024-06-11 19:29:16,828 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.21 seconds +2024-06-11T19:29:17.548926673Z 2024-06-11 19:29:17,548 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.18 seconds +2024-06-11T19:29:20.590152008Z 2024-06-11 19:29:20,589 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.76 seconds +2024-06-11T19:29:20.620573383Z 2024-06-11 19:29:20,620 - URLSemantic - INFO - ⏰ total elapsed time: 32.66 seconds +2024-06-11T19:29:20.620608841Z 2024-06-11 19:29:20,620 - URLSemantic - INFO - 📖 number of docs analyzed: 9 +2024-06-11T19:29:20.629547841Z 2024-06-11 19:29:20,628 - __main__ - INFO - 1 +2024-06-11T19:29:20.631334633Z 2024-06-11 19:29:20,630 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 9 +2024-06-11T19:29:20.631374841Z 2024-06-11 19:29:20,630 - __main__ - INFO - ⏰⏰ total elapsed time: 32.71 seconds +2024-06-11T19:29:20.631378091Z 2024-06-11 19:29:20,630 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.867.867.1718132806355894184.141', data=b'\n+https://help.collaboard.app/de/gastbenutzer(\x81\x80\x9c\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:29:20.643851550Z 2024-06-11 19:29:20,643 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.868.868.1718132806357045850.140', data=b'\n.https://help.collaboard.app/move-on-the-canvas(\x81\x80\x9e\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:29:20.643866925Z 2024-06-11 19:29:20,643 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:29:20.643870550Z 2024-06-11 19:29:20,643 - __main__ - INFO - message: +2024-06-11T19:29:20.643872716Z url: "https://help.collaboard.app/move-on-the-canvas" +2024-06-11T19:29:20.643885383Z document_id: 976690891547181057 +2024-06-11T19:29:20.643888216Z connector_id: 976341399530373121 +2024-06-11T19:29:20.643889716Z file_type: URL +2024-06-11T19:29:20.643891050Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:29:20.643892675Z +2024-06-11T19:29:20.643896091Z 2024-06-11 19:29:20,643 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:29:20.643899425Z 2024-06-11 19:29:20,643 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:29:20.681418341Z 2024-06-11 19:29:20,681 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:29:20.681434091Z 2024-06-11 19:29:20,681 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/move-on-the-canvas +2024-06-11T19:29:20.681435675Z 2024-06-11 19:29:20,681 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/move-on-the-canvas +2024-06-11T19:29:20.800720258Z 2024-06-11 19:29:20,800 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:29:20.812418050Z 2024-06-11 19:29:20,812 - Milvus_DB - INFO - deleting all entities related to document 976690891547181057 +2024-06-11T19:29:20.825107091Z 2024-06-11 19:29:20,824 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:29:20.833640050Z 2024-06-11 19:29:20,833 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:29:20.833648716Z 2024-06-11 19:29:20,833 - root - INFO - splitting the entity +2024-06-11T19:29:20.833650216Z 2024-06-11 19:29:20,833 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:29:20.833883550Z 2024-06-11 19:29:20,833 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/move-on-the-canvas +2024-06-11T19:29:20.833886508Z 2024-06-11 19:29:20,833 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:29:21.456466175Z 2024-06-11 19:29:21,456 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:29:24.507212385Z 2024-06-11 19:29:24,507 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:29:25.145295927Z 2024-06-11 19:29:25,145 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:29:27.743272761Z 2024-06-11 19:29:27,743 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.23 seconds +2024-06-11T19:29:28.434404178Z 2024-06-11 19:29:28,434 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.16 seconds +2024-06-11T19:29:31.510252763Z 2024-06-11 19:29:31,509 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.76 seconds +2024-06-11T19:29:32.121148847Z 2024-06-11 19:29:32,120 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:29:35.174301848Z 2024-06-11 19:29:35,173 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:29:35.793192557Z 2024-06-11 19:29:35,793 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:29:38.846450725Z 2024-06-11 19:29:38,846 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.66 seconds +2024-06-11T19:29:39.534252350Z 2024-06-11 19:29:39,534 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.14 seconds +2024-06-11T19:29:42.580060727Z 2024-06-11 19:29:42,579 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.73 seconds +2024-06-11T19:29:43.172347977Z 2024-06-11 19:29:43,172 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:29:46.223844770Z 2024-06-11 19:29:46,223 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:29:46.875785312Z 2024-06-11 19:29:46,875 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.11 seconds +2024-06-11T19:29:49.993878480Z 2024-06-11 19:29:49,993 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.76 seconds +2024-06-11T19:29:50.620162022Z 2024-06-11 19:29:50,619 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:29:54.185184691Z 2024-06-11 19:29:54,184 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.17 seconds +2024-06-11T19:29:54.828720024Z 2024-06-11 19:29:54,828 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.10 seconds +2024-06-11T19:29:57.903182817Z 2024-06-11 19:29:57,902 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:29:57.915032526Z 2024-06-11 19:29:57,914 - URLSemantic - INFO - ⏰ total elapsed time: 37.23 seconds +2024-06-11T19:29:57.915044067Z 2024-06-11 19:29:57,914 - URLSemantic - INFO - 📖 number of docs analyzed: 10 +2024-06-11T19:29:57.920989859Z 2024-06-11 19:29:57,920 - __main__ - INFO - 1 +2024-06-11T19:29:57.922757526Z 2024-06-11 19:29:57,922 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 10 +2024-06-11T19:29:57.922766859Z 2024-06-11 19:29:57,922 - __main__ - INFO - ⏰⏰ total elapsed time: 37.28 seconds +2024-06-11T19:29:57.922769151Z 2024-06-11 19:29:57,922 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.868.868.1718132806357045850.140', data=b'\n.https://help.collaboard.app/move-on-the-canvas(\x81\x80\x9e\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:29:57.931088651Z 2024-06-11 19:29:57,930 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.869.869.1718132806358170309.139', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.869.869.1718132806358170309.139', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.870.870.1718132806359163809.138', data=b'\n6https://help.collaboard.app/zoom-in-and-out-the-canvas(\x81\x80\xa2\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:32:45.949185672Z 2024-06-11 19:32:45,948 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:32:45.949190089Z 2024-06-11 19:32:45,948 - __main__ - INFO - message: +2024-06-11T19:32:45.949191672Z url: "https://help.collaboard.app/zoom-in-and-out-the-canvas" +2024-06-11T19:32:45.949192839Z document_id: 976690891547246593 +2024-06-11T19:32:45.949194339Z connector_id: 976341399530373121 +2024-06-11T19:32:45.949195297Z file_type: URL +2024-06-11T19:32:45.949196172Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:32:45.949197297Z +2024-06-11T19:32:45.949199172Z 2024-06-11 19:32:45,948 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:32:45.949203255Z 2024-06-11 19:32:45,948 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:32:46.009444755Z 2024-06-11 19:32:46,009 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:32:46.009467922Z 2024-06-11 19:32:46,009 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/zoom-in-and-out-the-canvas +2024-06-11T19:32:46.009471797Z 2024-06-11 19:32:46,009 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/zoom-in-and-out-the-canvas +2024-06-11T19:32:46.197768880Z 2024-06-11 19:32:46,197 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.19 seconds +2024-06-11T19:32:46.213113339Z 2024-06-11 19:32:46,212 - Milvus_DB - INFO - deleting all entities related to document 976690891547246593 +2024-06-11T19:32:46.227299005Z 2024-06-11 19:32:46,227 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:32:46.238370505Z 2024-06-11 19:32:46,238 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:32:46.238385047Z 2024-06-11 19:32:46,238 - root - INFO - splitting the entity +2024-06-11T19:32:46.238395464Z 2024-06-11 19:32:46,238 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:32:46.238832755Z 2024-06-11 19:32:46,238 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/zoom-in-and-out-the-canvas +2024-06-11T19:32:46.238837964Z 2024-06-11 19:32:46,238 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:32:46.807548672Z 2024-06-11 19:32:46,807 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:32:49.874553590Z 2024-06-11 19:32:49,874 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:32:50.500864216Z 2024-06-11 19:32:50,500 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.09 seconds +2024-06-11T19:32:53.579053051Z 2024-06-11 19:32:53,578 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.70 seconds +2024-06-11T19:32:54.254893634Z 2024-06-11 19:32:54,254 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.11 seconds +2024-06-11T19:32:57.293236927Z 2024-06-11 19:32:57,293 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.69 seconds +2024-06-11T19:32:57.884029469Z 2024-06-11 19:32:57,883 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:33:00.954180304Z 2024-06-11 19:33:00,953 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:33:01.600078221Z 2024-06-11 19:33:01,599 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.11 seconds +2024-06-11T19:33:04.189240681Z 2024-06-11 19:33:04,188 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.22 seconds +2024-06-11T19:33:04.789423931Z 2024-06-11 19:33:04,789 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:33:07.867389349Z 2024-06-11 19:33:07,867 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:33:08.488586224Z 2024-06-11 19:33:08,488 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:33:11.558008309Z 2024-06-11 19:33:11,557 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.67 seconds +2024-06-11T19:33:12.267622143Z 2024-06-11 19:33:12,267 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.16 seconds +2024-06-11T19:33:14.865242644Z 2024-06-11 19:33:14,865 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.29 seconds +2024-06-11T19:33:15.468661311Z 2024-06-11 19:33:15,468 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:33:16.018345811Z 127.0.0.1 - - [11/Jun/2024 19:33:16] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:33:16.018422436Z 2024-06-11 19:33:16,018 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:33:18.529335979Z 2024-06-11 19:33:18,529 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:33:19.157630271Z 2024-06-11 19:33:19,157 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:33:22.242223814Z 2024-06-11 19:33:22,241 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.71 seconds +2024-06-11T19:33:22.273926439Z 2024-06-11 19:33:22,273 - URLSemantic - INFO - ⏰ total elapsed time: 36.26 seconds +2024-06-11T19:33:22.273966273Z 2024-06-11 19:33:22,273 - URLSemantic - INFO - 📖 number of docs analyzed: 10 +2024-06-11T19:33:22.282742314Z 2024-06-11 19:33:22,282 - __main__ - INFO - 1 +2024-06-11T19:33:22.285370148Z 2024-06-11 19:33:22,284 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 10 +2024-06-11T19:33:22.285382606Z 2024-06-11 19:33:22,284 - __main__ - INFO - ⏰⏰ total elapsed time: 36.34 seconds +2024-06-11T19:33:22.285385856Z 2024-06-11 19:33:22,284 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.870.870.1718132806359163809.138', data=b'\n6https://help.collaboard.app/zoom-in-and-out-the-canvas(\x81\x80\xa2\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:22.310305773Z 2024-06-11 19:33:22,309 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.871.871.1718132806360281517.137', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.871.871.1718132806360281517.137', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.873.873.1718132806362457225.135', data=b'\n7https://help.collaboard.app/de/dein-browser-ist-gezoomt(\x81\x80\xa8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:29.919369234Z 2024-06-11 19:33:29,918 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:29.919370984Z 2024-06-11 19:33:29,918 - __main__ - INFO - message: +2024-06-11T19:33:29.919372443Z url: "https://help.collaboard.app/de/dein-browser-ist-gezoomt" +2024-06-11T19:33:29.919373818Z document_id: 976690891547344897 +2024-06-11T19:33:29.919374943Z connector_id: 976341399530373121 +2024-06-11T19:33:29.919376068Z file_type: URL +2024-06-11T19:33:29.919377151Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:29.919378401Z +2024-06-11T19:33:29.919379484Z 2024-06-11 19:33:29,918 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:29.919380734Z 2024-06-11 19:33:29,918 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:29.957405901Z 2024-06-11 19:33:29,957 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:29.957422526Z 2024-06-11 19:33:29,957 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/dein-browser-ist-gezoomt +2024-06-11T19:33:29.957424360Z 2024-06-11 19:33:29,957 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/dein-browser-ist-gezoomt +2024-06-11T19:33:30.281398401Z 2024-06-11 19:33:30,281 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.32 seconds +2024-06-11T19:33:30.295883776Z 2024-06-11 19:33:30,295 - Milvus_DB - INFO - deleting all entities related to document 976690891547344897 +2024-06-11T19:33:30.307285485Z 2024-06-11 19:33:30,307 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:30.317191068Z 2024-06-11 19:33:30,317 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:30.317207985Z 2024-06-11 19:33:30,317 - root - INFO - splitting the entity +2024-06-11T19:33:30.317209901Z 2024-06-11 19:33:30,317 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:30.317462485Z 2024-06-11 19:33:30,317 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/dein-browser-ist-gezoomt +2024-06-11T19:33:30.317466526Z 2024-06-11 19:33:30,317 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:33:30.901915527Z 2024-06-11 19:33:30,901 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:33:33.957833445Z 2024-06-11 19:33:33,957 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:33:33.963353153Z 2024-06-11 19:33:33,963 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134411.888111985,0): "sql txn" meta={id=5979cfc8 key=/Table/121/1/976696160939769857/0 iso=Serializable pri=0.04214756 epo=0 ts=1718134412.119368027,1 min=1718134410.321571818,0 seq=1} lock=true stat=PENDING rts=1718134410.321571818,0 wto=false gul=1718134410.821571818,0 +2024-06-11T19:33:33.963361570Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:33.963363236Z +2024-06-11T19:33:33.963364278Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:33.963365361Z 2024-06-11 19:33:33,963 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:33.971347320Z 2024-06-11 19:33:33,970 - __main__ - INFO - 1 +2024-06-11T19:33:33.972705778Z 2024-06-11 19:33:33,972 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:33.972712528Z 2024-06-11 19:33:33,972 - __main__ - INFO - ⏰⏰ total elapsed time: 4.05 seconds +2024-06-11T19:33:33.972713945Z 2024-06-11 19:33:33,972 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.873.873.1718132806362457225.135', data=b'\n7https://help.collaboard.app/de/dein-browser-ist-gezoomt(\x81\x80\xa8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:33.981355570Z 2024-06-11 19:33:33,981 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.875.875.1718132806365031892.133', data=b'\nIhttps://help.collaboard.app/de/mehrere-objekte-ausw\xc3\xa4hlen-und-verschieben(\x81\x80\xac\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:33.981362195Z 2024-06-11 19:33:33,981 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:33.981363778Z 2024-06-11 19:33:33,981 - __main__ - INFO - message: +2024-06-11T19:33:33.981364903Z url: "https://help.collaboard.app/de/mehrere-objekte-auswählen-und-verschieben" +2024-06-11T19:33:33.981366153Z document_id: 976690891547410433 +2024-06-11T19:33:33.981367736Z connector_id: 976341399530373121 +2024-06-11T19:33:33.981368695Z file_type: URL +2024-06-11T19:33:33.981369611Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:33.981370653Z +2024-06-11T19:33:33.981371528Z 2024-06-11 19:33:33,981 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:33.981372570Z 2024-06-11 19:33:33,981 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:34.023680820Z 2024-06-11 19:33:34,022 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:34.023696778Z 2024-06-11 19:33:34,022 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/mehrere-objekte-auswählen-und-verschieben +2024-06-11T19:33:34.023698736Z 2024-06-11 19:33:34,022 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/mehrere-objekte-auswählen-und-verschieben +2024-06-11T19:33:34.174998612Z 2024-06-11 19:33:34,174 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.15 seconds +2024-06-11T19:33:34.187854570Z 2024-06-11 19:33:34,187 - Milvus_DB - INFO - deleting all entities related to document 976690891547410433 +2024-06-11T19:33:34.198595487Z 2024-06-11 19:33:34,198 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:34.209265862Z 2024-06-11 19:33:34,209 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:34.209274653Z 2024-06-11 19:33:34,209 - root - INFO - splitting the entity +2024-06-11T19:33:34.209276487Z 2024-06-11 19:33:34,209 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:34.209536903Z 2024-06-11 19:33:34,209 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/mehrere-objekte-auswählen-und-verschieben +2024-06-11T19:33:34.209540112Z 2024-06-11 19:33:34,209 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:33:34.785860862Z 2024-06-11 19:33:34,785 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:33:37.872434405Z 2024-06-11 19:33:37,872 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:33:37.878644905Z 2024-06-11 19:33:37,878 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134415.976998154,0): "sql txn" meta={id=5083ca9c key=/Table/121/1/976696173768998913/0 iso=Serializable pri=0.01700781 epo=0 ts=1718134416.166642029,1 min=1718134414.214971028,0 seq=1} lock=true stat=PENDING rts=1718134414.214971028,0 wto=false gul=1718134414.714971028,0 +2024-06-11T19:33:37.878653780Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:37.878655613Z +2024-06-11T19:33:37.878656738Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:37.878657822Z 2024-06-11 19:33:37,878 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:37.883871280Z 2024-06-11 19:33:37,883 - __main__ - INFO - 1 +2024-06-11T19:33:37.885723572Z 2024-06-11 19:33:37,885 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:37.885729697Z 2024-06-11 19:33:37,885 - __main__ - INFO - ⏰⏰ total elapsed time: 3.90 seconds +2024-06-11T19:33:37.885731072Z 2024-06-11 19:33:37,885 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.875.875.1718132806365031892.133', data=b'\nIhttps://help.collaboard.app/de/mehrere-objekte-ausw\xc3\xa4hlen-und-verschieben(\x81\x80\xac\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:37.886637863Z 2024-06-11 19:33:37,885 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.877.877.1718132806367926392.131', data=b'\nHhttps://help.collaboard.app/de/objekte-gruppieren-/-gruppierung-aufheben(\x81\x80\xb0\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:37.886643405Z 2024-06-11 19:33:37,885 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:37.886644822Z 2024-06-11 19:33:37,885 - __main__ - INFO - message: +2024-06-11T19:33:37.886645863Z url: "https://help.collaboard.app/de/objekte-gruppieren-/-gruppierung-aufheben" +2024-06-11T19:33:37.886647072Z document_id: 976690891547475969 +2024-06-11T19:33:37.886648030Z connector_id: 976341399530373121 +2024-06-11T19:33:37.886648988Z file_type: URL +2024-06-11T19:33:37.886650072Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:37.886651113Z +2024-06-11T19:33:37.886651947Z 2024-06-11 19:33:37,885 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:37.886652988Z 2024-06-11 19:33:37,885 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:37.919746030Z 2024-06-11 19:33:37,919 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:37.919754822Z 2024-06-11 19:33:37,919 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/objekte-gruppieren-/-gruppierung-aufheben +2024-06-11T19:33:37.919756322Z 2024-06-11 19:33:37,919 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/objekte-gruppieren-/-gruppierung-aufheben +2024-06-11T19:33:38.046352488Z 2024-06-11 19:33:38,046 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.13 seconds +2024-06-11T19:33:38.057764488Z 2024-06-11 19:33:38,057 - Milvus_DB - INFO - deleting all entities related to document 976690891547475969 +2024-06-11T19:33:38.069761697Z 2024-06-11 19:33:38,069 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:38.078929322Z 2024-06-11 19:33:38,078 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:38.078935530Z 2024-06-11 19:33:38,078 - root - INFO - splitting the entity +2024-06-11T19:33:38.078937613Z 2024-06-11 19:33:38,078 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:38.079064572Z 2024-06-11 19:33:38,078 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/objekte-gruppieren-/-gruppierung-aufheben +2024-06-11T19:33:38.079069155Z 2024-06-11 19:33:38,079 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:33:38.686558947Z 2024-06-11 19:33:38,686 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.08 seconds +2024-06-11T19:33:41.220654782Z 2024-06-11 19:33:41,220 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.14 seconds +2024-06-11T19:33:41.228789240Z 2024-06-11 19:33:41,228 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134420.025170531,0): "sql txn" meta={id=f5264fe4 key=/Table/121/1/976696184739233793/0 iso=Serializable pri=0.00865143 epo=0 ts=1718134420.210794739,1 min=1718134418.083684655,0 seq=1} lock=true stat=PENDING rts=1718134418.083684655,0 wto=false gul=1718134418.583684655,0 +2024-06-11T19:33:41.228798157Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:41.228799823Z +2024-06-11T19:33:41.228801115Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:41.228802323Z 2024-06-11 19:33:41,228 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:41.234518740Z 2024-06-11 19:33:41,234 - __main__ - INFO - 1 +2024-06-11T19:33:41.235503740Z 2024-06-11 19:33:41,235 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:41.235508532Z 2024-06-11 19:33:41,235 - __main__ - INFO - ⏰⏰ total elapsed time: 3.35 seconds +2024-06-11T19:33:41.235510157Z 2024-06-11 19:33:41,235 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.877.877.1718132806367926392.131', data=b'\nHhttps://help.collaboard.app/de/objekte-gruppieren-/-gruppierung-aufheben(\x81\x80\xb0\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:41.236541907Z 2024-06-11 19:33:41,236 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.879.879.1718132806370594684.129', data=b'\n=https://help.collaboard.app/de/objekte-sperren-und-entsperren(\x81\x80\xb4\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:41.236549198Z 2024-06-11 19:33:41,236 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:41.236551073Z 2024-06-11 19:33:41,236 - __main__ - INFO - message: +2024-06-11T19:33:41.236552490Z url: "https://help.collaboard.app/de/objekte-sperren-und-entsperren" +2024-06-11T19:33:41.236553782Z document_id: 976690891547541505 +2024-06-11T19:33:41.236554948Z connector_id: 976341399530373121 +2024-06-11T19:33:41.236556115Z file_type: URL +2024-06-11T19:33:41.236557240Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:41.236558490Z +2024-06-11T19:33:41.236564240Z 2024-06-11 19:33:41,236 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:41.236565490Z 2024-06-11 19:33:41,236 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:41.279756907Z 2024-06-11 19:33:41,279 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:41.279772323Z 2024-06-11 19:33:41,279 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/objekte-sperren-und-entsperren +2024-06-11T19:33:41.279774698Z 2024-06-11 19:33:41,279 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/objekte-sperren-und-entsperren +2024-06-11T19:33:41.412252740Z 2024-06-11 19:33:41,412 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.13 seconds +2024-06-11T19:33:41.427231740Z 2024-06-11 19:33:41,426 - Milvus_DB - INFO - deleting all entities related to document 976690891547541505 +2024-06-11T19:33:41.438832198Z 2024-06-11 19:33:41,438 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:41.448756282Z 2024-06-11 19:33:41,448 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:41.448763740Z 2024-06-11 19:33:41,448 - root - INFO - splitting the entity +2024-06-11T19:33:41.448765073Z 2024-06-11 19:33:41,448 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:41.449220365Z 2024-06-11 19:33:41,448 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/objekte-sperren-und-entsperren +2024-06-11T19:33:41.449224907Z 2024-06-11 19:33:41,448 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:33:42.038845907Z 2024-06-11 19:33:42,038 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:33:45.078075533Z 2024-06-11 19:33:45,077 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.62 seconds +2024-06-11T19:33:45.083146700Z 2024-06-11 19:33:45,083 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134424.053489825,0): "sql txn" meta={id=b9d7a36a key=/Table/121/1/976696197379293185/0 iso=Serializable pri=0.00163605 epo=0 ts=1718134424.270036325,1 min=1718134421.454208490,0 seq=1} lock=true stat=PENDING rts=1718134421.454208490,0 wto=false gul=1718134421.954208490,0 +2024-06-11T19:33:45.083156117Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:45.083158033Z +2024-06-11T19:33:45.083159283Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:45.083160492Z 2024-06-11 19:33:45,083 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:45.087794283Z 2024-06-11 19:33:45,087 - __main__ - INFO - 1 +2024-06-11T19:33:45.088979492Z 2024-06-11 19:33:45,088 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:45.088994158Z 2024-06-11 19:33:45,088 - __main__ - INFO - ⏰⏰ total elapsed time: 3.85 seconds +2024-06-11T19:33:45.088995992Z 2024-06-11 19:33:45,088 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.879.879.1718132806370594684.129', data=b'\n=https://help.collaboard.app/de/objekte-sperren-und-entsperren(\x81\x80\xb4\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:45.089849617Z 2024-06-11 19:33:45,089 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.881.881.1718132806373383309.127', data=b'\n/https://help.collaboard.app/de/objekte-anordner(\x81\x80\xb8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:45.089854242Z 2024-06-11 19:33:45,089 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:45.089855450Z 2024-06-11 19:33:45,089 - __main__ - INFO - message: +2024-06-11T19:33:45.089856492Z url: "https://help.collaboard.app/de/objekte-anordner" +2024-06-11T19:33:45.089857575Z document_id: 976690891547607041 +2024-06-11T19:33:45.089858658Z connector_id: 976341399530373121 +2024-06-11T19:33:45.089859533Z file_type: URL +2024-06-11T19:33:45.089860450Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:45.089861492Z +2024-06-11T19:33:45.089862367Z 2024-06-11 19:33:45,089 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:45.089863408Z 2024-06-11 19:33:45,089 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:45.125584033Z 2024-06-11 19:33:45,125 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:45.125601492Z 2024-06-11 19:33:45,125 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/objekte-anordner +2024-06-11T19:33:45.125602992Z 2024-06-11 19:33:45,125 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/objekte-anordner +2024-06-11T19:33:45.223512992Z 2024-06-11 19:33:45,223 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:33:45.237352950Z 2024-06-11 19:33:45,237 - Milvus_DB - INFO - deleting all entities related to document 976690891547607041 +2024-06-11T19:33:45.252751825Z 2024-06-11 19:33:45,252 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:45.265116992Z 2024-06-11 19:33:45,264 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:45.265127117Z 2024-06-11 19:33:45,264 - root - INFO - splitting the entity +2024-06-11T19:33:45.265128533Z 2024-06-11 19:33:45,264 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:45.265130075Z 2024-06-11 19:33:45,264 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/objekte-anordner +2024-06-11T19:33:45.265135533Z 2024-06-11 19:33:45,264 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:33:45.869862784Z 2024-06-11 19:33:45,869 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:33:48.924165910Z 2024-06-11 19:33:48,924 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:33:48.965422744Z 2024-06-11 19:33:48,964 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134428.087037451,0): "sql txn" meta={id=cc793a7d key=/Table/121/1/976696209981865985/0 iso=Serializable pri=0.00657705 epo=0 ts=1718134428.232277118,1 min=1718134425.272603575,0 seq=1} lock=true stat=PENDING rts=1718134425.272603575,0 wto=false gul=1718134425.772603575,0 +2024-06-11T19:33:48.965434869Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:48.965436660Z +2024-06-11T19:33:48.965438035Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:48.965439285Z 2024-06-11 19:33:48,964 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:49.006555369Z 2024-06-11 19:33:49,006 - __main__ - INFO - 1 +2024-06-11T19:33:49.007386160Z 2024-06-11 19:33:49,007 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:49.007631035Z 2024-06-11 19:33:49,007 - __main__ - INFO - ⏰⏰ total elapsed time: 3.92 seconds +2024-06-11T19:33:49.007789452Z 2024-06-11 19:33:49,007 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.881.881.1718132806373383309.127', data=b'\n/https://help.collaboard.app/de/objekte-anordner(\x81\x80\xb8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:49.008741327Z 2024-06-11 19:33:49,008 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.883.883.1718132806376059392.125', data=b'\nEhttps://help.collaboard.app/de/r\xc3\xbcckg\xc3\xa4ngig-machen-/-wiederherstellen(\x81\x80\xbc\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:49.008899952Z 2024-06-11 19:33:49,008 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:49.009261869Z 2024-06-11 19:33:49,009 - __main__ - INFO - message: +2024-06-11T19:33:49.009265869Z url: "https://help.collaboard.app/de/rückgängig-machen-/-wiederherstellen" +2024-06-11T19:33:49.009267535Z document_id: 976690891547672577 +2024-06-11T19:33:49.009268827Z connector_id: 976341399530373121 +2024-06-11T19:33:49.009269994Z file_type: URL +2024-06-11T19:33:49.009271202Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:49.009285702Z +2024-06-11T19:33:49.009425827Z 2024-06-11 19:33:49,009 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:49.010291744Z 2024-06-11 19:33:49,009 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:49.047177244Z 2024-06-11 19:33:49,046 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:49.047195910Z 2024-06-11 19:33:49,046 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/rückgängig-machen-/-wiederherstellen +2024-06-11T19:33:49.047197994Z 2024-06-11 19:33:49,047 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/rückgängig-machen-/-wiederherstellen +2024-06-11T19:33:49.181128660Z 2024-06-11 19:33:49,180 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.13 seconds +2024-06-11T19:33:49.196588327Z 2024-06-11 19:33:49,196 - Milvus_DB - INFO - deleting all entities related to document 976690891547672577 +2024-06-11T19:33:49.208699160Z 2024-06-11 19:33:49,208 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:49.219630660Z 2024-06-11 19:33:49,219 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:49.219637160Z 2024-06-11 19:33:49,219 - root - INFO - splitting the entity +2024-06-11T19:33:49.219638452Z 2024-06-11 19:33:49,219 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:49.219995869Z 2024-06-11 19:33:49,219 - URLSemantic - INFO - created 11 chunks for https://help.collaboard.app/de/rückgängig-machen-/-wiederherstellen +2024-06-11T19:33:49.220002244Z 2024-06-11 19:33:49,219 - root - INFO - saving in milvus 11 chunk(s) +2024-06-11T19:33:49.788946577Z 2024-06-11 19:33:49,788 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:33:52.835643412Z 2024-06-11 19:33:52,835 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:33:52.876251370Z 2024-06-11 19:33:52,876 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134432.083989537,0): "sql txn" meta={id=0a492007 key=/Table/121/1/976696222800084993/0 iso=Serializable pri=0.01779296 epo=0 ts=1718134432.273634662,1 min=1718134429.224647202,0 seq=1} lock=true stat=PENDING rts=1718134429.224647202,0 wto=false gul=1718134429.724647202,0 +2024-06-11T19:33:52.876274287Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:52.876275954Z +2024-06-11T19:33:52.876277037Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:52.877353620Z 2024-06-11 19:33:52,876 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:52.891618495Z 2024-06-11 19:33:52,891 - __main__ - INFO - 1 +2024-06-11T19:33:52.893174120Z 2024-06-11 19:33:52,892 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:52.893195620Z 2024-06-11 19:33:52,892 - __main__ - INFO - ⏰⏰ total elapsed time: 3.88 seconds +2024-06-11T19:33:52.893198120Z 2024-06-11 19:33:52,893 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.883.883.1718132806376059392.125', data=b'\nEhttps://help.collaboard.app/de/r\xc3\xbcckg\xc3\xa4ngig-machen-/-wiederherstellen(\x81\x80\xbc\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:52.894090620Z 2024-06-11 19:33:52,893 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.885.885.1718132806378629225.123', data=b'\n/https://help.collaboard.app/de/objekte-kopieren(\x81\x80\xc0\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:52.894145870Z 2024-06-11 19:33:52,894 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:52.894210662Z 2024-06-11 19:33:52,894 - __main__ - INFO - message: +2024-06-11T19:33:52.894215245Z url: "https://help.collaboard.app/de/objekte-kopieren" +2024-06-11T19:33:52.894216537Z document_id: 976690891547738113 +2024-06-11T19:33:52.894217912Z connector_id: 976341399530373121 +2024-06-11T19:33:52.894218870Z file_type: URL +2024-06-11T19:33:52.894219829Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:52.894220829Z +2024-06-11T19:33:52.894352495Z 2024-06-11 19:33:52,894 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:52.894795745Z 2024-06-11 19:33:52,894 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:52.947755245Z 2024-06-11 19:33:52,947 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:52.947771370Z 2024-06-11 19:33:52,947 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/objekte-kopieren +2024-06-11T19:33:52.947773454Z 2024-06-11 19:33:52,947 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/objekte-kopieren +2024-06-11T19:33:53.064963371Z 2024-06-11 19:33:53,064 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:33:53.081090621Z 2024-06-11 19:33:53,080 - Milvus_DB - INFO - deleting all entities related to document 976690891547738113 +2024-06-11T19:33:53.091943454Z 2024-06-11 19:33:53,091 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:53.102698621Z 2024-06-11 19:33:53,102 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:53.102709579Z 2024-06-11 19:33:53,102 - root - INFO - splitting the entity +2024-06-11T19:33:53.102710996Z 2024-06-11 19:33:53,102 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:53.102962787Z 2024-06-11 19:33:53,102 - URLSemantic - INFO - created 14 chunks for https://help.collaboard.app/de/objekte-kopieren +2024-06-11T19:33:53.102976704Z 2024-06-11 19:33:53,102 - root - INFO - saving in milvus 14 chunk(s) +2024-06-11T19:33:53.689026787Z 2024-06-11 19:33:53,688 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:33:57.256963831Z 2024-06-11 19:33:57,256 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 4.15 seconds +2024-06-11T19:33:57.262458289Z 2024-06-11 19:33:57,262 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134436.059284372,0): "sql txn" meta={id=88361095 key=/Table/121/1/976696237286719489/0 iso=Serializable pri=0.00575529 epo=0 ts=1718134436.288534164,1 min=1718134433.109804412,0 seq=1} lock=true stat=PENDING rts=1718134433.109804412,0 wto=false gul=1718134433.609804412,0 +2024-06-11T19:33:57.262468039Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:33:57.262469998Z +2024-06-11T19:33:57.262472206Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:33:57.262473664Z 2024-06-11 19:33:57,262 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:33:57.268225206Z 2024-06-11 19:33:57,267 - __main__ - INFO - 1 +2024-06-11T19:33:57.269376873Z 2024-06-11 19:33:57,269 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:33:57.269385539Z 2024-06-11 19:33:57,269 - __main__ - INFO - ⏰⏰ total elapsed time: 4.38 seconds +2024-06-11T19:33:57.269387456Z 2024-06-11 19:33:57,269 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.885.885.1718132806378629225.123', data=b'\n/https://help.collaboard.app/de/objekte-kopieren(\x81\x80\xc0\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:33:57.270288039Z 2024-06-11 19:33:57,269 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.887.887.1718132806381074892.121', data=b'\n.https://help.collaboard.app/de/links-erstellen(\x81\x80\xc4\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:33:57.270299623Z 2024-06-11 19:33:57,269 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:33:57.270301914Z 2024-06-11 19:33:57,269 - __main__ - INFO - message: +2024-06-11T19:33:57.270303498Z url: "https://help.collaboard.app/de/links-erstellen" +2024-06-11T19:33:57.270305039Z document_id: 976690891547803649 +2024-06-11T19:33:57.270306414Z connector_id: 976341399530373121 +2024-06-11T19:33:57.270307748Z file_type: URL +2024-06-11T19:33:57.270308998Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:33:57.270316414Z +2024-06-11T19:33:57.270317664Z 2024-06-11 19:33:57,269 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:33:57.270318873Z 2024-06-11 19:33:57,269 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:33:57.309697289Z 2024-06-11 19:33:57,309 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:33:57.309709081Z 2024-06-11 19:33:57,309 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/links-erstellen +2024-06-11T19:33:57.309710581Z 2024-06-11 19:33:57,309 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/links-erstellen +2024-06-11T19:33:57.416549956Z 2024-06-11 19:33:57,415 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.11 seconds +2024-06-11T19:33:57.427945456Z 2024-06-11 19:33:57,427 - Milvus_DB - INFO - deleting all entities related to document 976690891547803649 +2024-06-11T19:33:57.439943289Z 2024-06-11 19:33:57,439 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:33:57.448469623Z 2024-06-11 19:33:57,448 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:33:57.448526331Z 2024-06-11 19:33:57,448 - root - INFO - splitting the entity +2024-06-11T19:33:57.448531123Z 2024-06-11 19:33:57,448 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:33:57.448895623Z 2024-06-11 19:33:57,448 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/links-erstellen +2024-06-11T19:33:57.448901789Z 2024-06-11 19:33:57,448 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:33:58.059001498Z 2024-06-11 19:33:58,058 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds +2024-06-11T19:34:01.092698083Z 2024-06-11 19:34:01,092 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:34:01.105533499Z 2024-06-11 19:34:01,105 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134440.139756416,0): "sql txn" meta={id=02f4081b key=/Table/121/1/976696249854951425/0 iso=Serializable pri=0.01278627 epo=0 ts=1718134440.299683082,1 min=1718134437.453867748,0 seq=1} lock=true stat=PENDING rts=1718134437.453867748,0 wto=false gul=1718134437.953867748,0 +2024-06-11T19:34:01.105548166Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:34:01.105550083Z +2024-06-11T19:34:01.105551333Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:34:01.105552583Z 2024-06-11 19:34:01,105 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:34:01.110955374Z 2024-06-11 19:34:01,110 - __main__ - INFO - 1 +2024-06-11T19:34:01.112127291Z 2024-06-11 19:34:01,111 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:34:01.112140291Z 2024-06-11 19:34:01,111 - __main__ - INFO - ⏰⏰ total elapsed time: 3.84 seconds +2024-06-11T19:34:01.112142208Z 2024-06-11 19:34:01,111 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.887.887.1718132806381074892.121', data=b'\n.https://help.collaboard.app/de/links-erstellen(\x81\x80\xc4\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:34:01.113622583Z 2024-06-11 19:34:01,112 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.889.889.1718132806383876809.119', data=b'\n?https://help.collaboard.app/de/anpinnen-und-l\xc3\xb6sen-von-objekten(\x81\x80\xc8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:34:01.113632458Z 2024-06-11 19:34:01,112 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:34:01.113635083Z 2024-06-11 19:34:01,112 - __main__ - INFO - message: +2024-06-11T19:34:01.113636333Z url: "https://help.collaboard.app/de/anpinnen-und-lösen-von-objekten" +2024-06-11T19:34:01.113637458Z document_id: 976690891547869185 +2024-06-11T19:34:01.113638499Z connector_id: 976341399530373121 +2024-06-11T19:34:01.113639458Z file_type: URL +2024-06-11T19:34:01.113640416Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:34:01.113641458Z +2024-06-11T19:34:01.113642374Z 2024-06-11 19:34:01,112 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:34:01.113643416Z 2024-06-11 19:34:01,112 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:34:01.148133833Z 2024-06-11 19:34:01,147 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:34:01.148150708Z 2024-06-11 19:34:01,147 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/anpinnen-und-lösen-von-objekten +2024-06-11T19:34:01.148152624Z 2024-06-11 19:34:01,147 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/anpinnen-und-lösen-von-objekten +2024-06-11T19:34:01.283712749Z 2024-06-11 19:34:01,283 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.14 seconds +2024-06-11T19:34:01.296434374Z 2024-06-11 19:34:01,296 - Milvus_DB - INFO - deleting all entities related to document 976690891547869185 +2024-06-11T19:34:01.308634624Z 2024-06-11 19:34:01,308 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:34:01.319136291Z 2024-06-11 19:34:01,318 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:34:01.319154916Z 2024-06-11 19:34:01,318 - root - INFO - splitting the entity +2024-06-11T19:34:01.319156666Z 2024-06-11 19:34:01,318 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:34:01.319261333Z 2024-06-11 19:34:01,319 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/anpinnen-und-lösen-von-objekten +2024-06-11T19:34:01.319263791Z 2024-06-11 19:34:01,319 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:34:01.905358666Z 2024-06-11 19:34:01,905 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:34:04.979701001Z 2024-06-11 19:34:04,979 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.65 seconds +2024-06-11T19:34:05.013728918Z 2024-06-11 19:34:05,013 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134444.076677501,0): "sql txn" meta={id=8a0e6d8b key=/Table/121/1/976696262593511425/0 iso=Serializable pri=0.00391086 epo=0 ts=1718134444.259368209,1 min=1718134441.324565416,0 seq=1} lock=true stat=PENDING rts=1718134441.324565416,0 wto=false gul=1718134441.824565416,0 +2024-06-11T19:34:05.013754626Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:34:05.013759960Z +2024-06-11T19:34:05.013762043Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:34:05.013766960Z 2024-06-11 19:34:05,013 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:34:05.062107710Z 2024-06-11 19:34:05,062 - __main__ - INFO - 1 +2024-06-11T19:34:05.063857126Z 2024-06-11 19:34:05,063 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:34:05.063865043Z 2024-06-11 19:34:05,063 - __main__ - INFO - ⏰⏰ total elapsed time: 3.95 seconds +2024-06-11T19:34:05.063867085Z 2024-06-11 19:34:05,063 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.889.889.1718132806383876809.119', data=b'\n?https://help.collaboard.app/de/anpinnen-und-l\xc3\xb6sen-von-objekten(\x81\x80\xc8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:34:05.064424668Z 2024-06-11 19:34:05,063 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.891.891.1718132806386727975.117', data=b'\n%https://help.collaboard.app/de/karten(\x81\x80\xcc\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:34:05.064432501Z 2024-06-11 19:34:05,063 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:34:05.064436751Z 2024-06-11 19:34:05,063 - __main__ - INFO - message: +2024-06-11T19:34:05.064439543Z url: "https://help.collaboard.app/de/karten" +2024-06-11T19:34:05.064442751Z document_id: 976690891547934721 +2024-06-11T19:34:05.064443876Z connector_id: 976341399530373121 +2024-06-11T19:34:05.064444960Z file_type: URL +2024-06-11T19:34:05.064457543Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:34:05.064462085Z +2024-06-11T19:34:05.064464501Z 2024-06-11 19:34:05,064 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:34:05.064465668Z 2024-06-11 19:34:05,064 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:34:05.099043126Z 2024-06-11 19:34:05,098 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:34:05.099057710Z 2024-06-11 19:34:05,098 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/karten +2024-06-11T19:34:05.099118293Z 2024-06-11 19:34:05,098 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/karten +2024-06-11T19:34:05.325387918Z 2024-06-11 19:34:05,324 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.23 seconds +2024-06-11T19:34:05.345670876Z 2024-06-11 19:34:05,345 - Milvus_DB - INFO - deleting all entities related to document 976690891547934721 +2024-06-11T19:34:05.362700918Z 2024-06-11 19:34:05,362 - Milvus_DB - INFO - ⏰ total elapsed time: 0.02 seconds +2024-06-11T19:34:05.371855126Z 2024-06-11 19:34:05,371 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:34:05.371860501Z 2024-06-11 19:34:05,371 - root - INFO - splitting the entity +2024-06-11T19:34:05.371862001Z 2024-06-11 19:34:05,371 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:34:05.372196210Z 2024-06-11 19:34:05,371 - URLSemantic - INFO - created 12 chunks for https://help.collaboard.app/de/karten +2024-06-11T19:34:05.372201418Z 2024-06-11 19:34:05,371 - root - INFO - saving in milvus 12 chunk(s) +2024-06-11T19:34:05.965459002Z 2024-06-11 19:34:05,965 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:34:09.007232586Z 2024-06-11 19:34:09,007 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:34:09.047678128Z 2024-06-11 19:34:09,047 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134448.113757794,0): "sql txn" meta={id=eaf370f5 key=/Table/121/1/976696275790692353/0 iso=Serializable pri=0.05303412 epo=0 ts=1718134448.259966919,1 min=1718134445.376440418,0 seq=1} lock=true stat=PENDING rts=1718134445.376440418,0 wto=false gul=1718134445.876440418,0 +2024-06-11T19:34:09.047685003Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:34:09.047686545Z +2024-06-11T19:34:09.047687545Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:34:09.047688628Z 2024-06-11 19:34:09,047 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:34:09.051950670Z 2024-06-11 19:34:09,051 - __main__ - INFO - 1 +2024-06-11T19:34:09.053171878Z 2024-06-11 19:34:09,053 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:34:09.053177211Z 2024-06-11 19:34:09,053 - __main__ - INFO - ⏰⏰ total elapsed time: 3.99 seconds +2024-06-11T19:34:09.053178586Z 2024-06-11 19:34:09,053 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.891.891.1718132806386727975.117', data=b'\n%https://help.collaboard.app/de/karten(\x81\x80\xcc\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:34:09.054402211Z 2024-06-11 19:34:09,054 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.893.893.1718132806390135475.115', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.893.893.1718132806390135475.115', data=b'\n, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.895.895.1718132806392899767.113', data=b'\n=https://help.collaboard.app/de/arbeiten-mit-collaboard/karten(\x81\x80\xd4\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:34:13.043964505Z 2024-06-11 19:34:13,043 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:34:13.043966922Z 2024-06-11 19:34:13,043 - __main__ - INFO - message: +2024-06-11T19:34:13.043968213Z url: "https://help.collaboard.app/de/arbeiten-mit-collaboard/karten" +2024-06-11T19:34:13.043969338Z document_id: 976690891548065793 +2024-06-11T19:34:13.043970380Z connector_id: 976341399530373121 +2024-06-11T19:34:13.043976922Z file_type: URL +2024-06-11T19:34:13.043978005Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:34:13.043979047Z +2024-06-11T19:34:13.043979963Z 2024-06-11 19:34:13,043 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:34:13.043981005Z 2024-06-11 19:34:13,043 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:34:13.083123422Z 2024-06-11 19:34:13,082 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:34:13.083136422Z 2024-06-11 19:34:13,082 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/arbeiten-mit-collaboard/karten +2024-06-11T19:34:13.083138047Z 2024-06-11 19:34:13,082 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/arbeiten-mit-collaboard/karten +2024-06-11T19:34:13.597215089Z 2024-06-11 19:34:13,596 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.51 seconds +2024-06-11T19:34:13.617599297Z 2024-06-11 19:34:13,617 - Milvus_DB - INFO - deleting all entities related to document 976690891548065793 +2024-06-11T19:34:13.631528714Z 2024-06-11 19:34:13,631 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:34:13.642844880Z 2024-06-11 19:34:13,642 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:34:13.642850672Z 2024-06-11 19:34:13,642 - root - INFO - splitting the entity +2024-06-11T19:34:13.642852047Z 2024-06-11 19:34:13,642 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:34:13.643193630Z 2024-06-11 19:34:13,642 - URLSemantic - INFO - created 13 chunks for https://help.collaboard.app/de/arbeiten-mit-collaboard/karten +2024-06-11T19:34:13.643198255Z 2024-06-11 19:34:13,642 - root - INFO - saving in milvus 13 chunk(s) +2024-06-11T19:34:14.217534881Z 2024-06-11 19:34:14,217 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.04 seconds +2024-06-11T19:34:16.138169173Z 127.0.0.1 - - [11/Jun/2024 19:34:16] "GET /healthz HTTP/1.1" 200 - +2024-06-11T19:34:16.138439923Z 2024-06-11 19:34:16,137 - ReadinessProbeHandler - INFO - /healthz response 200 +2024-06-11T19:34:17.257900799Z 2024-06-11 19:34:17,257 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.61 seconds +2024-06-11T19:34:17.295573465Z 2024-06-11 19:34:17,295 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134456.127586382,0): "sql txn" meta={id=2f347b64 key=/Table/121/1/976696302843494401/0 iso=Serializable pri=0.02595279 epo=0 ts=1718134456.652938840,1 min=1718134453.647720047,0 seq=1} lock=true stat=PENDING rts=1718134453.647720047,0 wto=false gul=1718134454.147720047,0 +2024-06-11T19:34:17.295582757Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:34:17.295585715Z +2024-06-11T19:34:17.295610465Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:34:17.295612674Z 2024-06-11 19:34:17,295 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:34:17.316917257Z 2024-06-11 19:34:17,316 - __main__ - INFO - 1 +2024-06-11T19:34:17.318041507Z 2024-06-11 19:34:17,317 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:34:17.318046715Z 2024-06-11 19:34:17,317 - __main__ - INFO - ⏰⏰ total elapsed time: 4.27 seconds +2024-06-11T19:34:17.318048174Z 2024-06-11 19:34:17,317 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.895.895.1718132806392899767.113', data=b'\n=https://help.collaboard.app/de/arbeiten-mit-collaboard/karten(\x81\x80\xd4\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:34:17.319672257Z 2024-06-11 19:34:17,319 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.897.897.1718132806395647434.111', data=b'\n9https://help.collaboard.app/extract-pages-from-a-document(\x81\x80\xd8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:34:17.319678674Z 2024-06-11 19:34:17,319 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:34:17.319680632Z 2024-06-11 19:34:17,319 - __main__ - INFO - message: +2024-06-11T19:34:17.319681632Z url: "https://help.collaboard.app/extract-pages-from-a-document" +2024-06-11T19:34:17.319682799Z document_id: 976690891548131329 +2024-06-11T19:34:17.319683715Z connector_id: 976341399530373121 +2024-06-11T19:34:17.319684590Z file_type: URL +2024-06-11T19:34:17.319685507Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:34:17.319686549Z +2024-06-11T19:34:17.319687424Z 2024-06-11 19:34:17,319 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:34:17.319688382Z 2024-06-11 19:34:17,319 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:34:17.409449257Z 2024-06-11 19:34:17,408 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:34:17.409465840Z 2024-06-11 19:34:17,408 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/extract-pages-from-a-document +2024-06-11T19:34:17.409467424Z 2024-06-11 19:34:17,408 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/extract-pages-from-a-document +2024-06-11T19:34:17.510102632Z 2024-06-11 19:34:17,509 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.10 seconds +2024-06-11T19:34:17.523916091Z 2024-06-11 19:34:17,523 - Milvus_DB - INFO - deleting all entities related to document 976690891548131329 +2024-06-11T19:34:17.534953341Z 2024-06-11 19:34:17,534 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:34:17.546344174Z 2024-06-11 19:34:17,546 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:34:17.546372466Z 2024-06-11 19:34:17,546 - root - INFO - splitting the entity +2024-06-11T19:34:17.546772299Z 2024-06-11 19:34:17,546 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:34:17.546777882Z 2024-06-11 19:34:17,546 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/extract-pages-from-a-document +2024-06-11T19:34:17.546779382Z 2024-06-11 19:34:17,546 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:34:18.139609382Z 2024-06-11 19:34:18,139 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.05 seconds +2024-06-11T19:34:21.185271842Z 2024-06-11 19:34:21,184 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.63 seconds +2024-06-11T19:34:21.193740759Z 2024-06-11 19:34:21,193 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134460.468861967,0): "sql txn" meta={id=c956602e key=/Table/121/1/976696315696578561/0 iso=Serializable pri=0.00813524 epo=0 ts=1718134460.625493925,1 min=1718134457.552254049,0 seq=1} lock=true stat=PENDING rts=1718134457.552254049,0 wto=false gul=1718134458.052254049,0 +2024-06-11T19:34:21.193750676Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:34:21.193752551Z +2024-06-11T19:34:21.193754092Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:34:21.193755509Z 2024-06-11 19:34:21,193 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:34:21.203510176Z 2024-06-11 19:34:21,202 - __main__ - INFO - 1 +2024-06-11T19:34:21.205455509Z 2024-06-11 19:34:21,204 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:34:21.205460884Z 2024-06-11 19:34:21,204 - __main__ - INFO - ⏰⏰ total elapsed time: 3.88 seconds +2024-06-11T19:34:21.205462509Z 2024-06-11 19:34:21,204 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.897.897.1718132806395647434.111', data=b'\n9https://help.collaboard.app/extract-pages-from-a-document(\x81\x80\xd8\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:34:21.205995634Z 2024-06-11 19:34:21,205 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.899.899.1718132806398358017.109', data=b'\nChttps://help.collaboard.app/de/anpinnen-und-l%C3%B6sen-von-objekten(\x81\x80\xdc\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:34:21.206002092Z 2024-06-11 19:34:21,205 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:34:21.206003759Z 2024-06-11 19:34:21,205 - __main__ - INFO - message: +2024-06-11T19:34:21.206010926Z url: "https://help.collaboard.app/de/anpinnen-und-l%C3%B6sen-von-objekten" +2024-06-11T19:34:21.206012384Z document_id: 976690891548196865 +2024-06-11T19:34:21.206013551Z connector_id: 976341399530373121 +2024-06-11T19:34:21.206014634Z file_type: URL +2024-06-11T19:34:21.206015842Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:34:21.206017134Z +2024-06-11T19:34:21.206258301Z 2024-06-11 19:34:21,205 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:34:21.206260967Z 2024-06-11 19:34:21,205 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:34:21.255363801Z 2024-06-11 19:34:21,255 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:34:21.255387051Z 2024-06-11 19:34:21,255 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/anpinnen-und-l%C3%B6sen-von-objekten +2024-06-11T19:34:21.255390426Z 2024-06-11 19:34:21,255 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/anpinnen-und-l%C3%B6sen-von-objekten +2024-06-11T19:34:21.380327176Z 2024-06-11 19:34:21,380 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.12 seconds +2024-06-11T19:34:21.393103301Z 2024-06-11 19:34:21,393 - Milvus_DB - INFO - deleting all entities related to document 976690891548196865 +2024-06-11T19:34:21.406476217Z 2024-06-11 19:34:21,406 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:34:21.443061176Z 2024-06-11 19:34:21,442 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:34:21.443106926Z 2024-06-11 19:34:21,442 - root - INFO - splitting the entity +2024-06-11T19:34:21.443109426Z 2024-06-11 19:34:21,443 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:34:21.443264426Z 2024-06-11 19:34:21,443 - URLSemantic - INFO - created 9 chunks for https://help.collaboard.app/de/anpinnen-und-l%C3%B6sen-von-objekten +2024-06-11T19:34:21.443268634Z 2024-06-11 19:34:21,443 - root - INFO - saving in milvus 9 chunk(s) +2024-06-11T19:34:22.053598093Z 2024-06-11 19:34:22,053 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.06 seconds +2024-06-11T19:34:25.108505261Z 2024-06-11 19:34:25,108 - Milvus_DB - INFO - ⏰ store into vector db, including embedding generation, total elapsed time: 3.64 seconds +2024-06-11T19:34:25.119994219Z 2024-06-11 19:34:25,119 - URLSemantic - ERROR - ❌ Failed to process semantic data: (psycopg2.errors.SerializationFailure) restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE - failed preemptive refresh due to encountered recently written committed value /Table/112/1/976341399530373121/0 @1718134464.437940677,0): "sql txn" meta={id=d2db6aee key=/Table/121/1/976696328553857025/0 iso=Serializable pri=0.00314084 epo=0 ts=1718134464.615287302,1 min=1718134461.470075842,0 seq=1} lock=true stat=PENDING rts=1718134461.470075842,0 wto=false gul=1718134461.970075842,0 +2024-06-11T19:34:25.120007427Z HINT: See: https://www.cockroachlabs.com/docs/v23.2/transaction-retry-error-reference.html#retry_serializable +2024-06-11T19:34:25.120017427Z +2024-06-11T19:34:25.120019177Z (Background on this error at: https://sqlalche.me/e/20/e3q8) +2024-06-11T19:34:25.120020761Z 2024-06-11 19:34:25,119 - __main__ - ERROR - ❌ entities_analyzed is none!!!!! +2024-06-11T19:34:25.127990136Z 2024-06-11 19:34:25,127 - __main__ - INFO - 1 +2024-06-11T19:34:25.129176677Z 2024-06-11 19:34:25,128 - __main__ - INFO - 👍 message acknowledged successfully, total entities stored 0 +2024-06-11T19:34:25.129184344Z 2024-06-11 19:34:25,128 - __main__ - INFO - ⏰⏰ total elapsed time: 3.92 seconds +2024-06-11T19:34:25.129186344Z 2024-06-11 19:34:25,128 - JetStreamEventSubscriber - INFO - Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.899.899.1718132806398358017.109', data=b'\nChttps://help.collaboard.app/de/anpinnen-und-l%C3%B6sen-von-objekten(\x81\x80\xdc\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=True, _sid=2) +2024-06-11T19:34:25.130759386Z 2024-06-11 19:34:25,130 - JetStreamEventSubscriber - INFO - [Msg(_client=, subject='semantic_activity', reply='$JS.ACK.semantic.worker.1.901.901.1718132806401628600.107', data=b'\n2https://help.collaboard.app/de/dokumente-hochladen(\x81\x80\xe0\xf0\x8e\xe6\xf9\xc6\r0\x81\x80\x90\xd4\xc7\xaa\xaa\xc6\r8\x01B%user_625ece7e042d4f40bd2588b16bec7be6', headers=None, _metadata=None, _ackd=False, _sid=2)] +2024-06-11T19:34:25.130765969Z 2024-06-11 19:34:25,130 - __main__ - INFO - 🔥 starting semantic analysis.. +2024-06-11T19:34:25.130768052Z 2024-06-11 19:34:25,130 - __main__ - INFO - message: +2024-06-11T19:34:25.130769427Z url: "https://help.collaboard.app/de/dokumente-hochladen" +2024-06-11T19:34:25.130770886Z document_id: 976690891548262401 +2024-06-11T19:34:25.130772094Z connector_id: 976341399530373121 +2024-06-11T19:34:25.130773219Z file_type: URL +2024-06-11T19:34:25.130774344Z collection_name: "user_625ece7e042d4f40bd2588b16bec7be6" +2024-06-11T19:34:25.130775677Z +2024-06-11T19:34:25.130776719Z 2024-06-11 19:34:25,130 - __main__ - ERROR - ❌ no model nameeeeeeee +2024-06-11T19:34:25.130778094Z 2024-06-11 19:34:25,130 - __main__ - WARNING - 😱 Addning model name and dimension manually remove this code ASAP +2024-06-11T19:34:25.165006261Z 2024-06-11 19:34:25,164 - URLSemantic - INFO - Analyzing URL +2024-06-11T19:34:25.165060636Z 2024-06-11 19:34:25,164 - URLSemantic - INFO - Starting BS4Spider URL: https://help.collaboard.app/de/dokumente-hochladen +2024-06-11T19:34:25.165065094Z 2024-06-11 19:34:25,164 - lib.spider.spider_bs4 - INFO - Processing URL: https://help.collaboard.app/de/dokumente-hochladen +2024-06-11T19:34:25.413408178Z 2024-06-11 19:34:25,413 - lib.spider.spider_bs4 - INFO - ⏰ total elapsed time: 0.25 seconds +2024-06-11T19:34:25.428698678Z 2024-06-11 19:34:25,428 - Milvus_DB - INFO - deleting all entities related to document 976690891548262401 +2024-06-11T19:34:25.438841094Z 2024-06-11 19:34:25,438 - Milvus_DB - INFO - ⏰ total elapsed time: 0.01 seconds +2024-06-11T19:34:25.447466969Z 2024-06-11 19:34:25,447 - root - INFO - storing in milvus 1 entities. One entity might be split in several chunks +2024-06-11T19:34:25.447481344Z 2024-06-11 19:34:25,447 - root - INFO - splitting the entity +2024-06-11T19:34:25.447483136Z 2024-06-11 19:34:25,447 - root - WARNING - 😱 split_data shall implement various chunk techniques and compare them +2024-06-11T19:34:25.447735719Z 2024-06-11 19:34:25,447 - URLSemantic - INFO - created 10 chunks for https://help.collaboard.app/de/dokumente-hochladen +2024-06-11T19:34:25.447739386Z 2024-06-11 19:34:25,447 - root - INFO - saving in milvus 10 chunk(s) +2024-06-11T19:34:26.057147470Z 2024-06-11 19:34:26,057 - Milvus_DB - INFO - ⏰ total elapsed time to create embedding: 0.07 seconds diff --git a/src/backend/semantic/test_cockroach.py b/src/backend/semantic/test_cockroach.py new file mode 100644 index 00000000..1477acbf --- /dev/null +++ b/src/backend/semantic/test_cockroach.py @@ -0,0 +1,99 @@ +import asyncio +import logging +import os +import uuid + +from dotenv import load_dotenv +from sqlalchemy import func + +from lib.db.db_document import DocumentCRUD +from lib.db.db_connector import ConnectorCRUD, Status + +# Load environment variables from .env file +load_dotenv() + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + +# loading from env env +nats_url = os.getenv('NATS_CLIENT_URL', 'nats://127.0.0.1:4222') +nats_connect_timeout = int(os.getenv('NATS_CLIENT_CONNECT_TIMEOUT', '30')) +nats_reconnect_time_wait = int(os.getenv('NATS_CLIENT_RECONNECT_TIME_WAIT', '30')) +nats_max_reconnect_attempts = int(os.getenv('NATS_CLIENT_MAX_RECONNECT_ATTEMPTS', '3')) +semantic_stream_name = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_NAME', 'semantic') +semantic_stream_subject = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_SUBJECT', 'chunk_activity') +semantic_ack_wait = int(os.getenv('NATS_CLIENT_SEMANTIC_ACK_WAIT', '3600')) # seconds +semantic_max_deliver = int(os.getenv('NATS_CLIENT_SEMANTIC_MAX_DELIVER', '3')) + +cockroach_url = os.getenv('COCKROACH_CLIENT_DATABASE_URL', + 'cockroachdb://root:123@cockroach:26257/defaultdb?sslmode=disable') + + +async def main(): + logger.info(cockroach_url) + + crud = DocumentCRUD(cockroach_url) + # Insert a new document + # new_doc_id = crud.insert_document( + # parent_id=None, + # connector_id=1, + # source_id='unique_source_id', + # url='http://example.com', + # signature='signature_example', + # chunking_session=uuid.uuid4(), + # analyzed=False, + # creation_date=func.now(), + # last_update=None + # ) + # print(f"Inserted document ID: {new_doc_id}") + + # Select the document + document = crud.select_document(974396356851630081) + logger.info(f"Selected document: {document}") + + # # Update the document + # crud.update_document(new_doc_id, url='http://newexample.com') + # + # # Delete the document + # crud.delete_document(new_doc_id) + # print(f"Deleted document ID: {new_doc_id}") + + # # Connector operations + connector_crud = ConnectorCRUD(cockroach_url) + # new_connector_id = connector_crud.insert_connector( + # credential_id=None, + # name='Connector Name', + # type='Connector Type', + # connector_specific_config={}, + # refresh_freq=3600, + # user_id=uuid.uuid4(), + # tenant_id=None, + # disabled=False, + # last_successful_index_date=None, + # last_attempt_status=None, + # total_docs_indexed=0, + # creation_date=func.now(), + # last_update=None, + # deleted_date=None + # ) + # print(f"Inserted connector ID: {new_connector_id}") + + connector = connector_crud.select_connector(document.connector_id) + logger.info(f"Selected connector: {connector}") + connector_crud.update_connector(connector.id, last_attempt_status=Status.SCAN_COMPLETED_SUCCESSFULLY) + connector = connector_crud.select_connector(document.connector_id) + logger.info(f"Selected connector: {connector}") + + # connector_crud.delete_connector(new_connector_id) + # print(f"Deleted connector ID: {new_connector_id}") + # + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/backend/semantic/test_log.py b/src/backend/semantic/test_log.py new file mode 100644 index 00000000..c6e003df --- /dev/null +++ b/src/backend/semantic/test_log.py @@ -0,0 +1,27 @@ +import asyncio +import logging +import os +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +# Get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) + +# Get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s') + +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + +def test_logging(): + logger.info("This is a test log message") + +if __name__ == "__main__": + logger.info("Starting main script") + test_logging() + logger.info("Completed test logging") + diff --git a/src/backend/semantic/test_mistune.py b/src/backend/semantic/test_mistune.py new file mode 100644 index 00000000..469df552 --- /dev/null +++ b/src/backend/semantic/test_mistune.py @@ -0,0 +1,102 @@ +import os +import sys +import time +import pymupdf4llm +import mistune +from typing import List, Dict, Any + + +class MarkdownSectionExtractor: + def __init__(self): + pass + + def _parse_markdown_content(self, markdown_content: str) -> List[Dict[str, Any]]: + markdown = mistune.create_markdown(renderer='ast') + return markdown(markdown_content) + + def extract_sections(self, markdown_content: str) -> List[Dict[str, Any]]: + parsed_content = self._parse_markdown_content(markdown_content) + sections = [] + current_section = [] + heading_hierarchy = [] + + def add_section(): + if current_section: + sections.append({ + 'headings': heading_hierarchy[:], # Copy of current heading hierarchy + 'content': current_section[:] + }) + current_section.clear() + + for element in parsed_content: + if element['type'] == 'heading': + level = element['attrs']['level'] + text = element['children'][0]['raw'] if element['children'] else '' + add_section() # Finish the current section before starting a new heading + + # Update heading hierarchy based on the level + if level == 1: + heading_hierarchy = [text] + else: + if len(heading_hierarchy) >= level: + heading_hierarchy = heading_hierarchy[:level - 1] + heading_hierarchy.append(text) + + current_section.append(element) + else: + current_section.append(element) + + add_section() # Add the last section + + return sections + + def extract_chunks(self, markdown: str) -> list[str]: + sections = self.extract_sections(markdown) + results = [] + # Process the extracted sections + for idx, section in enumerate(sections): + content_texts = [] + for element in section['content']: + if 'children' in element and element['children']: + for child in element['children']: + if 'raw' in child: + content_texts.append(child['raw']) + elif 'text' in child: + content_texts.append(child['text']) + elif 'raw' in element: + content_texts.append(element['raw']) + elif 'text' in element: + content_texts.append(element['text']) + + readable_text = ' '.join(content_texts) + headings_text = ' > '.join(section['headings']) + result = f"{headings_text}\n{readable_text}\n" + results.append(result) + return results + +if __name__ == "__main__": + import pathlib + + try: + filename = sys.argv[1] + except IndexError: + print(f"Usage:\npython {os.path.basename(__file__)} input.pdf") + sys.exit() + + t0 = time.perf_counter() + + markdown_content = pymupdf4llm.to_markdown(filename) + # print(markdown_content) + + extractor = MarkdownSectionExtractor() + # sections = extractor.extract_sections(markdown_content) + results = extractor.extract_chunks(markdown_content) + + + # Print the results + for result in results: + print(result) + print("****************** new section *********************\n") + + t1 = time.perf_counter() + print(f"Markdown creation time for {filename=} {round(t1 - t0, 2)} sec.") diff --git a/src/backend/semantic/test_publisher_2.py b/src/backend/semantic/test_publisher_2.py new file mode 100644 index 00000000..4e2cc649 --- /dev/null +++ b/src/backend/semantic/test_publisher_2.py @@ -0,0 +1,110 @@ +import asyncio +from nats.aio.client import Client as NATS + +from lib.db.jetstream_publisher import JetStreamPublisher +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.gen_types.file_type_pb2 import FileType +from nats.errors import TimeoutError, NoRespondersError +from nats.js.api import StreamConfig, RetentionPolicy +from nats.js.errors import BadRequestError +import logging +import os +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +nats_url = os.getenv('NATS_CLIENT_URL', 'nats://nats:4222') +nats_connect_timeout = int(os.getenv('NATS_CLIENT_CONNECT_TIMEOUT', '30')) +nats_reconnect_time_wait = int(os.getenv('NATS_CLIENT_RECONNECT_TIME_WAIT', '30')) +nats_max_reconnect_attempts = int(os.getenv('NATS_CLIENT_MAX_RECONNECT_ATTEMPTS', '3')) +semantic_stream_name = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_NAME', 'semantic') +semantic_stream_subject = os.getenv('NATS_CLIENT_SEMANTIC_STREAM_SUBJECT', 'semantic_activity') +semantic_ack_wait = int(os.getenv('NATS_CLIENT_SEMANTIC_ACK_WAIT', '3600')) # seconds +semantic_max_deliver = int(os.getenv('NATS_CLIENT_SEMANTIC_MAX_DELIVER', '3')) + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + + +def main(): + # Instantiate the publisher + logger.info(f"{semantic_stream_name} - {semantic_stream_subject}") + publisher = JetStreamPublisher(subject=semantic_stream_subject, stream_name=semantic_stream_name) + + # Connect to NATS + publisher.connect() + # semntic_data = SemanticData(url="minio:tenant-c20a9f75-a363-40ea-86ef-eabcedbac7df:0493eb3a-4475-462e-a791-e47834ea7ba8-small.pdf", + # document_id=976345414660126765, + # connector_id= 975493320735424513, + # file_type= FileType.PDF, + # collection_name= "user_625ece7e042d4f40bd2588b16bec7be6") + + # Create a fake ChunkingData message + semntic_data = SemanticData( + url="https://help.collaboard.app/sticky-notes", + # url = "https://developer.apple.com/documentation/visionos/improving-accessibility-support-in-your-app", + # url = "https://help.collaboard.app/what-is-collaboard", + # url = "https://learn.microsoft.com/en-us/aspnet/core/tutorials/razor-pages/?view=aspnetcore-8.0", + # url = "https://learn.microsoft.com/en-us/aspnet/core/tutorials/razor-pages/sql?view=aspnetcore-8.0&tabs=visual-studio", + site_map="", + url_recursive=False, + search_for_sitemap=True, + document_id=974396356851630081, + file_type=FileType.URL, + collection_name="user_id_998", + model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + model_dimension=768 + ) + + logger.info(f"message being sent \n {semntic_data}") + logger.info(f"{semantic_stream_name} - {semantic_stream_subject}") + # Publish the message + publisher.publish(semntic_data) + + # # Create a fake ChunkingData message + # chunking_data = SemanticData( + # url="https://help.collaboard.app/extract-pages-from-a-document", + # site_map="", + # search_for_sitemap=True, + # document_id=993456788, + # file_type=FileType.URL, + # collection_name="tennant_id_3", + # model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + # model_dimension=768 + # ) + + # logger.info(f"message being sent \n {chunking_data}") + + # # Publish the message + # await publisher.publish(chunking_data) + + # # Create a fake ChunkingData message + # chunking_data = SemanticData( + # url="https://help.collaboard.app/upload-images", + # site_map="", + # search_for_sitemap=True, + # document_id=993456788, + # file_type=FileType.URL, + # collection_name="tennant_id_5", + # model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + # model_dimension=768 + # ) + + # logger.info(f"message being sent \n {chunking_data}") + + # # Publish the message + # await publisher.publish(chunking_data) + + # Close the connection + publisher.close() + + +if __name__ == "__main__": + main() diff --git a/src/backend/semantic/test_pypdf.py b/src/backend/semantic/test_pypdf.py new file mode 100644 index 00000000..1cd0a20f --- /dev/null +++ b/src/backend/semantic/test_pypdf.py @@ -0,0 +1,584 @@ + + +import os +import string +import pymupdf +# try: +# import pymupdf as fitz # available with v1.24.3 +# except ImportError: +# import fitz + +from pymupdf4llm.helpers.get_text_lines import get_raw_lines, is_white +from pymupdf4llm.helpers.multi_column import column_boxes + +if pymupdf.pymupdf_version_tuple < (1, 24, 2): + raise NotImplementedError("PyMuPDF version 1.24.2 or later is needed.") + +bullet = ("- ", "* ", chr(0xF0A7), chr(0xF0B7), chr(0xB7), chr(8226), chr(9679)) +GRAPHICS_TEXT = "\n![%s](%s)\n" + + +class IdentifyHeaders: + """Compute data for identifying header text.""" + + def __init__( + self, + doc: str, + pages: list = None, + body_limit: float = 12, + ): + """Read all text and make a dictionary of fontsizes. + + Args: + pages: optional list of pages to consider + body_limit: consider text with larger font size as some header + """ + if isinstance(doc, pymupdf.Document): + mydoc = doc + else: + mydoc = pymupdf.open(doc) + + if pages is None: # use all pages if omitted + pages = range(mydoc.page_count) + + fontsizes = {} + for pno in pages: + page = mydoc.load_page(pno) + blocks = page.get_text("dict", flags=pymupdf.TEXTFLAGS_TEXT)["blocks"] + for span in [ # look at all non-empty horizontal spans + s + for b in blocks + for l in b["lines"] + for s in l["spans"] + if not is_white(s["text"]) + ]: + fontsz = round(span["size"]) + count = fontsizes.get(fontsz, 0) + len(span["text"].strip()) + fontsizes[fontsz] = count + + if mydoc != doc: + # if opened here, close it now + mydoc.close() + + # maps a fontsize to a string of multiple # header tag characters + self.header_id = {} + + # If not provided, choose the most frequent font size as body text. + # If no text at all on all pages, just use 12. + # In any case all fonts not exceeding + temp = sorted( + [(k, v) for k, v in fontsizes.items()], + key=lambda i: i[1], + reverse=True, + ) + if temp: + b_limit = max(body_limit, temp[0][0]) + else: + b_limit = body_limit + + # identify up to 6 font sizes as header candidates + sizes = sorted( + [f for f in fontsizes.keys() if f > b_limit], + reverse=True, + )[:6] + + # make the header tag dictionary + for i, size in enumerate(sizes): + self.header_id[size] = "#" * (i + 1) + " " + + def get_header_id(self, span: dict, page=None) -> str: + """Return appropriate markdown header prefix. + + Given a text span from a "dict"/"rawdict" extraction, determine the + markdown header prefix string of 0 to n concatenated '#' characters. + """ + fontsize = round(span["size"]) # compute fontsize + hdr_id = self.header_id.get(fontsize, "") + return hdr_id + + +def to_markdown( + doc: str, + *, + pages: list = None, + hdr_info=None, + write_images: bool = False, + page_chunks: bool = False, + margins=(0, 50, 0, 50), +) -> str: + """Process the document and return the text of its selected pages.""" + + if isinstance(doc, str): + doc = pymupdf.open(doc) + + if pages is None: # use all pages if no selection given + pages = list(range(doc.page_count)) + + if hasattr(margins, "__float__"): + margins = [margins] * 4 + if len(margins) == 2: + margins = (0, margins[0], 0, margins[1]) + if len(margins) != 4: + raise ValueError("margins must have length 2 or 4 or be a number.") + elif not all([hasattr(m, "__float__") for m in margins]): + raise ValueError("margin values must be numbers") + + # If "hdr_info" is not an object having method "get_header_id", scan the + # document and use font sizes as header level indicators. + if callable(hdr_info): + get_header_id = hdr_info + elif hasattr(hdr_info, "get_header_id") and callable(hdr_info.get_header_id): + get_header_id = hdr_info.get_header_id + else: + hdr_info = IdentifyHeaders(doc) + get_header_id = hdr_info.get_header_id + + def resolve_links(links, span): + """Accept a span and return a markdown link string.""" + bbox = pymupdf.Rect(span["bbox"]) # span bbox + # a link should overlap at least 70% of the span + bbox_area = 0.7 * abs(bbox) + for link in links: + hot = link["from"] # the hot area of the link + if not abs(hot & bbox) >= bbox_area: + continue # does not touch the bbox + text = f'[{span["text"].strip()}]({link["uri"]})' + return text + + def save_image(page, rect, i): + """Optionally render the rect part of a page.""" + filename = page.parent.name.replace("\\", "/") + image_path = f"{filename}-{page.number}-{i}.png" + if write_images is True: + pix = page.get_pixmap(clip=rect) + pix.save(image_path) + del pix + return os.path.basename(image_path) + return "" + + def write_text( + page: pymupdf.Page, + textpage: pymupdf.TextPage, + clip: pymupdf.Rect, + tabs=None, + tab_rects: dict = None, + img_rects: dict = None, + links: list = None, + ) -> string: + """Output the text found inside the given clip. + + This is an alternative for plain text in that it outputs + text enriched with markdown styling. + The logic is capable of recognizing headers, body text, code blocks, + inline code, bold, italic and bold-italic styling. + There is also some effort for list supported (ordered / unordered) in + that typical characters are replaced by respective markdown characters. + + 'tab_rects'/'img_rects' are dictionaries of table, respectively image + or vector graphic rectangles. + General Markdown text generation skips these areas. Tables are written + via their own 'to_markdown' method. Images and vector graphics are + optionally saved as files and pointed to by respective markdown text. + """ + if clip is None: + clip = textpage.rect + out_string = "" + + # This is a list of tuples (linerect, spanlist) + nlines = get_raw_lines(textpage, clip=clip, tolerance=3) + + tab_rects0 = list(tab_rects.values()) + img_rects0 = list(img_rects.values()) + + prev_lrect = None # previous line rectangle + prev_bno = -1 # previous block number of line + code = False # mode indicator: outputting code + prev_hdr_string = None + + for lrect, spans in nlines: + # there may tables or images inside the text block: skip them + if intersects_rects(lrect, tab_rects0) or intersects_rects( + lrect, img_rects0 + ): + continue + + # Pick up tables intersecting this text block + for i, tab_rect in sorted( + [ + j + for j in tab_rects.items() + if j[1].y1 <= lrect.y0 and not (j[1] & clip).is_empty + ], + key=lambda j: (j[1].y1, j[1].x0), + ): + out_string += "\n" + tabs[i].to_markdown(clean=False) + "\n" + del tab_rects[i] + + # Pick up images / graphics intersecting this text block + for i, img_rect in sorted( + [ + j + for j in img_rects.items() + if j[1].y1 <= lrect.y0 and not (j[1] & clip).is_empty + ], + key=lambda j: (j[1].y1, j[1].x0), + ): + pathname = save_image(page, img_rect, i) + if pathname: + out_string += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + + text = " ".join([s["text"] for s in spans]) + + # if the full line mono-spaced? + all_mono = all([s["flags"] & 8 for s in spans]) + + if all_mono: + if not code: # if not already in code output mode: + out_string += "```\n" # switch on "code" mode + code = True + # compute approx. distance from left - assuming a width + # of 0.5*fontsize. + delta = int((lrect.x0 - clip.x0) / (spans[0]["size"] * 0.5)) + indent = " " * delta + + out_string += indent + text + "\n" + continue # done with this line + + span0 = spans[0] + bno = span0["block"] # block number of line + if bno != prev_bno: + out_string += "\n" + prev_bno = bno + + if ( # check if we need another line break + prev_lrect + and lrect.y1 - prev_lrect.y1 > lrect.height * 1.5 + or span0["text"].startswith("[") + or span0["text"].startswith(bullet) + or span0["flags"] & 1 # superscript? + ): + out_string += "\n" + prev_lrect = lrect + + # if line is a header, this will return multiple "#" characters + hdr_string = get_header_id(span0, page=page) + + # intercept if header text has been broken in multiple lines + if hdr_string and hdr_string == prev_hdr_string: + out_string = out_string[:-1] + " " + text + "\n" + continue + + prev_hdr_string = hdr_string + if hdr_string.startswith("#"): # if a header line skip the rest + out_string += hdr_string + text + "\n" + continue + + # this line is not all-mono, so switch off "code" mode + if code: # still in code output mode? + out_string += "```\n" # switch of code mode + code = False + + for i, s in enumerate(spans): # iterate spans of the line + # decode font properties + mono = s["flags"] & 8 + bold = s["flags"] & 16 + italic = s["flags"] & 2 + + if mono: + # this is text in some monospaced font + out_string += f"`{s['text'].strip()}` " + else: # not a mono text + prefix = "" + suffix = "" + if hdr_string == "": + if bold: + prefix = "**" + suffix += "**" + if italic: + prefix += "_" + suffix = "_" + suffix + + # convert intersecting link into markdown syntax + ltext = resolve_links(links, s) + if ltext: + text = f"{hdr_string}{prefix}{ltext}{suffix} " + else: + text = f"{hdr_string}{prefix}{s['text'].strip()}{suffix} " + + if text.startswith(bullet): + text = "- " + text[1:] + out_string += text + if not code: + out_string += "\n" + out_string += "\n" + if code: + out_string += "```\n" # switch of code mode + code = False + + return ( + out_string.replace(" \n", "\n").replace(" ", " ").replace("\n\n\n", "\n\n") + ) + + def is_in_rects(rect, rect_list): + """Check if rect is contained in a rect of the list.""" + for i, r in enumerate(rect_list, start=1): + if rect in r: + return i + return 0 + + def intersects_rects(rect, rect_list): + """Check if middle of rect is contained in a rect of the list.""" + for i, r in enumerate(rect_list, start=1): + if (rect.tl + rect.br) / 2 in r: # middle point is inside r + return i + return 0 + + def output_tables(tabs, text_rect, tab_rects): + """Output tables above a text rectangle.""" + this_md = "" # markdown string for table content + if text_rect is not None: # select tables above the text block + for i, trect in sorted( + [j for j in tab_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0), + ): + this_md += tabs[i].to_markdown(clean=False) + del tab_rects[i] # do not touch this table twice + + else: # output all remaining table + for i, trect in sorted( + tab_rects.items(), + key=lambda j: (j[1].y1, j[1].x0), + ): + this_md += tabs[i].to_markdown(clean=False) + del tab_rects[i] # do not touch this table twice + return this_md + + def output_images(page, text_rect, img_rects): + """Output images and graphics above text rectangle.""" + if img_rects is None: + return "" + this_md = "" # markdown string + if text_rect is not None: # select tables above the text block + for i, img_rect in sorted( + [j for j in img_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0), + ): + pathname = save_image(page, img_rect, i) + if pathname: + this_md += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] # do not touch this image twice + + else: # output all remaining table + for i, img_rect in sorted( + img_rects.items(), + key=lambda j: (j[1].y1, j[1].x0), + ): + pathname = save_image(page, img_rect, i) + if pathname: + this_md += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] # do not touch this image twice + return this_md + + def get_metadata(doc, pno): + meta = doc.metadata.copy() + meta["file_path"] = doc.name + meta["page_count"] = doc.page_count + meta["page"] = pno + 1 + return meta + + def get_page_output(doc, pno, margins, textflags): + """Process one page. + + Args: + doc: fitz.Document + pno: 0-based page number + textflags: text extraction flag bits + + Returns: + Markdown string of page content and image, table and vector + graphics information. + """ + page = doc[pno] + md_string = "" + left, top, right, bottom = margins + clip = page.rect + (left, top, -right, -bottom) + # extract all links on page + links = [l for l in page.get_links() if l["kind"] == 2] + + # make a TextPage for all later extractions + textpage = page.get_textpage(flags=textflags, clip=clip) + + img_info = [img for img in page.get_image_info() if img["bbox"] in clip] + images = img_info[:] + tables = [] + graphics = [] + + # Locate all tables on page + tabs = page.find_tables(clip=clip, strategy="lines_strict") + + # Make a list of table boundary boxes. + # Must include the header bbox (may exist outside tab.bbox) + tab_rects = {} + for i, t in enumerate(tabs): + tab_rects[i] = pymupdf.Rect(t.bbox) | pymupdf.Rect(t.header.bbox) + tab_dict = { + "bbox": tuple(tab_rects[i]), + "rows": t.row_count, + "columns": t.col_count, + } + tables.append(tab_dict) + tab_rects0 = list(tab_rects.values()) + + # Select paths that are not contained in any table + page_clip = page.rect + (36, 36, -36, -36) # ignore full page graphics + paths = [ + p + for p in page.get_drawings() + if not intersects_rects(p["rect"], tab_rects0) + and p["rect"] in page_clip + and p["rect"].width < page_clip.width + and p["rect"].height < page_clip.height + ] + + # Determine vector graphics outside any tables, filerting out any + # which contain no stroked paths + vg_clusters = [] + for bbox in page.cluster_drawings(drawings=paths): + include = False + for p in [p for p in paths if p["rect"] in bbox]: + if p["type"] != "f": + include = True + break + if [item[0] for item in p["items"] if item[0] == "c"]: + include = True + break + if include is True: + vg_clusters.append(bbox) + + actual_paths = [p for p in paths if is_in_rects(p["rect"], vg_clusters)] + + vg_clusters0 = [ + r + for r in vg_clusters + if not intersects_rects(r, tab_rects0) and r.height > 20 + ] + + if write_images is True: + vg_clusters0 += [pymupdf.Rect(i["bbox"]) for i in img_info] + + vg_clusters = dict((i, r) for i, r in enumerate(vg_clusters0)) + + # Determine text column bboxes on page, avoiding tables and graphics + text_rects = column_boxes( + page, + paths=actual_paths, + no_image_text=write_images, + textpage=textpage, + avoid=tab_rects0 + vg_clusters0, + ) + """Extract markdown text iterating over text rectangles. + We also output any tables. They may live above, below or inside + the text rectangles. + """ + for text_rect in text_rects: + # output tables above this block of text + md_string += output_tables(tabs, text_rect, tab_rects) + md_string += output_images(page, text_rect, vg_clusters) + + # output text inside this rectangle + md_string += write_text( + page, + textpage, + text_rect, + tabs=tabs, + tab_rects=tab_rects, + img_rects=vg_clusters, + links=links, + ) + + # write any remaining tables and images + md_string += output_tables(tabs, None, tab_rects) + md_string += output_images(None, tab_rects, None) + md_string += "\n-----\n\n" + while md_string.startswith("\n"): + md_string = md_string[1:] + return md_string, images, tables, graphics + + if page_chunks is False: + document_output = "" + else: + document_output = [] + + # read the Table of Contents + toc = doc.get_toc() + textflags = pymupdf.TEXT_DEHYPHENATE | pymupdf.TEXT_MEDIABOX_CLIP + for pno in pages: + + page_output, images, tables, graphics = get_page_output( + doc, pno, margins, textflags + ) + if page_chunks is False: + document_output += page_output + else: + # build subet of TOC for this page + page_tocs = [t for t in toc if t[-1] == pno + 1] + + metadata = get_metadata(doc, pno) + document_output.append( + { + "metadata": metadata, + "toc_items": page_tocs, + "tables": tables, + "images": images, + "graphics": graphics, + "text": page_output, + } + ) + + return document_output + + +if __name__ == "__main__": + import pathlib + import sys + import time + + try: + filename = sys.argv[1] + except IndexError: + print(f"Usage:\npython {os.path.basename(__file__)} input.pdf") + sys.exit() + + t0 = time.perf_counter() # start a time + + doc = pymupdf.open(filename) # open input file + parms = sys.argv[2:] # contains ["-pages", "PAGES"] or empty list + pages = range(doc.page_count) # default page range + + # needed if we want to pass the number of pages to parse + # if len(parms) == 2 and parms[0] == "-pages": # page sub-selection given + # pages = [] # list of desired page numbers + # + # # replace any variable "N" by page count + # pages_spec = parms[1].replace("N", f"{doc.page_count}") + # for spec in pages_spec.split(","): + # if "-" in spec: + # start, end = map(int, spec.split("-")) + # pages.extend(range(start - 1, end)) + # else: + # pages.append(int(spec) - 1) + # + # # make a set of invalid page numbers + # wrong_pages = set([n + 1 for n in pages if n >= doc.page_count][:4]) + # if wrong_pages != set(): # if any invalid numbers given, exit. + # sys.exit(f"Page number(s) {wrong_pages} not in '{doc}'.") + + # get the markdown string + md_string = to_markdown(doc, pages=pages) + print(md_string) + # output to a text file with extension ".md" + outname = doc.name.replace(".pdf", ".md") + pathlib.Path(outname).write_bytes(md_string.encode()) + t1 = time.perf_counter() # stop timer + print(f"Markdown creation time for {doc.name=} {round(t1-t0,2)} sec.") \ No newline at end of file diff --git a/src/backend/semantic/test_pypdf_2.py b/src/backend/semantic/test_pypdf_2.py new file mode 100644 index 00000000..383a6474 --- /dev/null +++ b/src/backend/semantic/test_pypdf_2.py @@ -0,0 +1,67 @@ +import os +import os +import string +import pathlib +import sys +import time +import pymupdf + + +def extract_text_from_pdf(pdf_path): + # open document + doc = pymupdf.open(pdf_path) + + text = "" # we will return this string + row_count = 0 # counts table rows + header = "" # overall table header: output this only once! + + # iterate over the pages + for page in doc: + # only read the table rows on each page, ignore other content + tables = page.find_tables() # a "TableFinder" object + for table in tables: + + # on first page extract external column names if present + if page.number == 0 and table.header.external: + # build the overall table header string + # technical note: incomplete / complex tables may have + # "None" in some header cells. Just use empty string then. + header = ( + ";".join( + [ + name if name is not None else "" + for name in table.header.names + ] + ) + + "\n" + ) + text += header + row_count += 1 # increase row counter + + # output the table body + for row in table.extract(): # iterate over the table rows + + # again replace any "None" in cells by an empty string + row_text = ( + ";".join([cell if cell is not None else "" for cell in row]) + "\n" + ) + if row_text != header: # omit duplicates of header row + text += row_text + row_count += 1 # increase row counter + doc.close() # close document + print(f"Loaded {row_count} table rows from file '{doc.name}'.\n") + return text + + +if __name__ == "__main__": + import pathlib + import sys + import time + + try: + filename = sys.argv[1] + except IndexError: + print(f"Usage:\npython {os.path.basename(__file__)} input.pdf") + sys.exit() + + print(extract_text_from_pdf(filename)) diff --git a/src/backend/semantic/test_pypdf_3.py b/src/backend/semantic/test_pypdf_3.py new file mode 100644 index 00000000..6e97effe --- /dev/null +++ b/src/backend/semantic/test_pypdf_3.py @@ -0,0 +1,350 @@ +import os +import pymupdf +from pymupdf4llm.helpers.get_text_lines import get_raw_lines, is_white +from pymupdf4llm.helpers.multi_column import column_boxes +from typing import List, Tuple, Dict, Union, Any + +if pymupdf.pymupdf_version_tuple < (1, 24, 2): + raise NotImplementedError("PyMuPDF version 1.24.2 or later is needed.") + +bullet = ("- ", "* ", chr(0xF0A7), chr(0xF0B7), chr(0xB7), chr(8226), chr(9679)) +GRAPHICS_TEXT = "\n![%s](%s)\n" + + +class IdentifyHeaders: + def __init__(self, doc: Union[str, pymupdf.Document], pages: List[int] = None, body_limit: float = 12) -> None: + if isinstance(doc, pymupdf.Document): + mydoc = doc + else: + mydoc = pymupdf.open(doc) + + if pages is None: + pages = list(range(mydoc.page_count)) + + fontsizes: Dict[int, int] = {} + for pno in pages: + page = mydoc.load_page(pno) + blocks = page.get_text("dict", flags=pymupdf.TEXTFLAGS_TEXT)["blocks"] + for span in [s for b in blocks for l in b["lines"] for s in l["spans"] if not is_white(s["text"])]: + fontsz = round(span["size"]) + count = fontsizes.get(fontsz, 0) + len(span["text"].strip()) + fontsizes[fontsz] = count + + if mydoc != doc: + mydoc.close() + + self.header_id: Dict[int, str] = {} + temp = sorted([(k, v) for k, v in fontsizes.items()], key=lambda i: i[1], reverse=True) + if temp: + b_limit = max(body_limit, temp[0][0]) + else: + b_limit = body_limit + + sizes = sorted([f for f in fontsizes.keys() if f > b_limit], reverse=True)[:6] + + for i, size in enumerate(sizes): + self.header_id[size] = "#" * (i + 1) + " " + + def get_header_id(self, span: dict, page: pymupdf.Page = None) -> str: + fontsize = round(span["size"]) + return self.header_id.get(fontsize, "") + + +class MarkdownConverter: + def __init__(self, doc_path: str, pages: List[int] = None, body_limit: float = 12, write_images: bool = False, + page_chunks: bool = False, + margins: Union[float, Tuple[float, float], Tuple[float, float, float, float]] = ( + 0, 50, 0, 50)) -> None: + self.doc_path = doc_path + self.doc = pymupdf.open(doc_path) + self.pages = pages if pages is not None else list(range(self.doc.page_count)) + self.body_limit = body_limit + self.write_images = write_images + self.page_chunks = page_chunks + self.margins = self._validate_margins(margins) + self.hdr_info = IdentifyHeaders(self.doc, pages=self.pages, body_limit=body_limit) + + def _validate_margins(self, margins: Union[float, Tuple[float, float], Tuple[float, float, float, float]]) -> Tuple[ + float, float, float, float]: + if hasattr(margins, "__float__"): + return (margins, margins, margins, margins) + if len(margins) == 2: + return (0, margins[0], 0, margins[1]) + if len(margins) != 4: + raise ValueError("Margins must have length 2 or 4 or be a number.") + if not all([hasattr(m, "__float__") for m in margins]): + raise ValueError("Margin values must be numbers") + return tuple(margins) + + def convert_to_markdown(self) -> Union[str, List[Dict[str, Any]]]: + document_output: Union[str, List[Dict[str, Any]]] = "" if not self.page_chunks else [] + toc = self.doc.get_toc() + textflags = pymupdf.TEXT_DEHYPHENATE | pymupdf.TEXT_MEDIABOX_CLIP + + for pno in self.pages: + page_output, images, tables, graphics = self._get_page_output(pno, textflags) + if not self.page_chunks: + document_output += page_output + else: + page_tocs = [t for t in toc if t[-1] == pno + 1] + metadata = self._get_metadata(pno) + document_output.append({ + "metadata": metadata, + "toc_items": page_tocs, + "tables": tables, + "images": images, + "graphics": graphics, + "text": page_output, + }) + + return document_output + + def _get_metadata(self, pno: int) -> Dict[str, Union[str, int]]: + meta = self.doc.metadata.copy() + meta["file_path"] = self.doc.name + meta["page_count"] = self.doc.page_count + meta["page"] = pno + 1 + return meta + + def _get_page_output(self, pno: int, textflags: int) -> Tuple[ + str, List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]: + page = self.doc[pno] + md_string = "" + left, top, right, bottom = self.margins + clip = page.rect + (left, top, -right, -bottom) + links = [l for l in page.get_links() if l["kind"] == 2] + textpage = page.get_textpage(flags=textflags, clip=clip) + img_info = [img for img in page.get_image_info() if img["bbox"] in clip] + images = img_info[:] + tables = [] + graphics = [] + tabs = page.find_tables(clip=clip, strategy="lines_strict") + tab_rects = self._get_tab_rects(tabs) + tab_rects0 = list(tab_rects.values()) + paths, vg_clusters0 = self._get_paths_and_graphics(page, tab_rects0, clip, img_info) + vg_clusters = {i: r for i, r in enumerate(vg_clusters0)} + text_rects = column_boxes(page, paths=paths, no_image_text=self.write_images, textpage=textpage, + avoid=tab_rects0 + vg_clusters0) + + for text_rect in text_rects: + md_string += self._output_tables(tabs, text_rect, tab_rects) + md_string += self._output_images(page, text_rect, vg_clusters) + md_string += self._write_text(page, textpage, text_rect, tabs, tab_rects, vg_clusters, links) + + md_string += self._output_tables(tabs, None, tab_rects) + md_string += self._output_images(page, None, vg_clusters) + md_string += "\n-----\n\n" + while md_string.startswith("\n"): + md_string = md_string[1:] + + return md_string, images, tables, graphics + + def _get_tab_rects(self, tabs: List[Any]) -> Dict[int, pymupdf.Rect]: + tab_rects: Dict[int, pymupdf.Rect] = {} + for i, t in enumerate(tabs): + tab_rects[i] = pymupdf.Rect(t.bbox) | pymupdf.Rect(t.header.bbox) + return tab_rects + + def _get_paths_and_graphics(self, page: pymupdf.Page, tab_rects0: List[pymupdf.Rect], clip: pymupdf.Rect, + img_info: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[pymupdf.Rect]]: + page_clip = page.rect + (36, 36, -36, -36) + paths = [p for p in page.get_drawings() if + not self._intersects_rects(p["rect"], tab_rects0) and p["rect"] in page_clip and p[ + "rect"].width < page_clip.width and p["rect"].height < page_clip.height] + vg_clusters = [r for r in page.cluster_drawings(drawings=paths) if + not self._intersects_rects(r, tab_rects0) and r.height > 20] + + if self.write_images: + vg_clusters += [pymupdf.Rect(i["bbox"]) for i in img_info] + + return paths, vg_clusters + + def _intersects_rects(self, rect: pymupdf.Rect, rect_list: List[pymupdf.Rect]) -> bool: + for r in rect_list: + if (rect.tl + rect.br) / 2 in r: + return True + return False + + def _output_tables(self, tabs: List[Any], text_rect: pymupdf.Rect, tab_rects: Dict[int, pymupdf.Rect]) -> str: + this_md = "" + if text_rect is not None: + for i, trect in sorted([(j[0], j[1]) for j in tab_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0)): + this_md += tabs[i].to_markdown(clean=False) + del tab_rects[i] + else: + for i, trect in sorted(tab_rects.items(), key=lambda j: (j[1].y1, j[1].x0)): + this_md += tabs[i].to_markdown(clean=False) + del tab_rects[i] + return this_md + + def _output_images(self, page: pymupdf.Page, text_rect: pymupdf.Rect, img_rects: Dict[int, pymupdf.Rect]) -> str: + if img_rects is None: + return "" + this_md = "" + if text_rect is not None: + for i, img_rect in sorted([(j[0], j[1]) for j in img_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0)): + pathname = self._save_image(page, img_rect, i) + if pathname: + this_md += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + else: + for i, img_rect in sorted(img_rects.items(), key=lambda j: (j[1].y1, j[1].x0)): + pathname = self._save_image(page, img_rect, i) + if pathname: + this_md += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + return this_md + + def _save_image(self, page: pymupdf.Page, rect: pymupdf.Rect, i: int) -> str: + filename = page.parent.name.replace("\\", "/") + image_path = f"{filename}-{page.number}-{i}.png" + if self.write_images: + pix = page.get_pixmap(clip=rect) + pix.save(image_path) + del pix + return os.path.basename(image_path) + return "" + + def _write_text(self, page: pymupdf.Page, textpage: pymupdf.TextPage, clip: pymupdf.Rect, tabs: List[Any], + tab_rects: Dict[int, pymupdf.Rect], img_rects: Dict[int, pymupdf.Rect], + links: List[Dict[str, Any]]) -> str: + out_string = "" + nlines = get_raw_lines(textpage, clip=clip, tolerance=3) + tab_rects0 = list(tab_rects.values()) + img_rects0 = list(img_rects.values()) + prev_lrect = None + prev_bno = -1 + code = False + prev_hdr_string = None + + for lrect, spans in nlines: + if self._intersects_rects(lrect, tab_rects0) or self._intersects_rects(lrect, img_rects0): + continue + + for i, tab_rect in sorted( + [j for j in tab_rects.items() if j[1].y1 <= lrect.y0 and not (j[1] & clip).is_empty], + key=lambda j: (j[1].y1, j[1].x0)): + out_string += "\n" + tabs[i].to_markdown(clean=False) + "\n" + del tab_rects[i] + + for i, img_rect in sorted( + [j for j in img_rects.items() if j[1].y1 <= lrect.y0 and not (j[1] & clip).is_empty], + key=lambda j: (j[1].y1, j[1].x0)): + pathname = self._save_image(page, img_rect, i) + if pathname: + out_string += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + + text = " ".join([s["text"] for s in spans]) + all_mono = all([s["flags"] & 8 for s in spans]) + + if all_mono: + if not code: + out_string += "```\n" + code = True + delta = int((lrect.x0 - clip.x0) / (spans[0]["size"] * 0.5)) + indent = " " * delta + out_string += indent + text + "\n" + continue + + span0 = spans[0] + bno = span0["block"] + if bno != prev_bno: + out_string += "\n" + prev_bno = bno + + if (prev_lrect and lrect.y1 - prev_lrect.y1 > lrect.height * 1.5 or span0["text"].startswith("[") or span0[ + "text"].startswith(bullet) or span0["flags"] & 1): + out_string += "\n" + prev_lrect = lrect + + hdr_string = self.hdr_info.get_header_id(span0, page=page) + if hdr_string and hdr_string == prev_hdr_string: + out_string = out_string[:-1] + " " + text + "\n" + continue + + prev_hdr_string = hdr_string + if hdr_string.startswith("#"): + out_string += hdr_string + text + "\n" + continue + + if code: + out_string += "```\n" + code = False + + for i, s in enumerate(spans): + mono = s["flags"] & 8 + bold = s["flags"] & 16 + italic = s["flags"] & 2 + + if mono: + out_string += f"`{s['text'].strip()}` " + else: + prefix = "" + suffix = "" + if hdr_string == "": + if bold: + prefix = "**" + suffix += "**" + if italic: + prefix += "_" + suffix = "_" + suffix + + ltext = self._resolve_links(links, s) + if ltext: + text = f"{hdr_string}{prefix}{ltext}{suffix} " + else: + text = f"{hdr_string}{prefix}{s['text'].strip()}{suffix} " + + if text.startswith(bullet): + text = "- " + text[1:] + out_string += text + + if not code: + out_string += "\n" + out_string += "\n" + if code: + out_string += "```\n" + code = False + + return out_string.replace(" \n", "\n").replace(" ", " ").replace("\n\n\n", "\n\n") + + def _resolve_links(self, links: List[Dict[str, Any]], span: Dict[str, Any]) -> Union[str, None]: + bbox = pymupdf.Rect(span["bbox"]) + bbox_area = 0.7 * abs(bbox) + for link in links: + hot = link["from"] + if abs(hot & bbox) >= bbox_area: + return f'[{span["text"].strip()}]({link["uri"]})' + return None + + +if __name__ == "__main__": + import pathlib + import sys + import time + + try: + filename = sys.argv[1] + except IndexError: + print(f"Usage:\npython {os.path.basename(__file__)} input.pdf") + sys.exit() + + t0 = time.perf_counter() + converter = MarkdownConverter(filename) + md_string = converter.convert_to_markdown() + + if isinstance(md_string, str): + print(md_string) + outname = converter.doc_path.replace(".pdf", ".md") + pathlib.Path(outname).write_bytes(md_string.encode()) + else: + for page in md_string: + print(page["text"]) + outname = converter.doc_path.replace(".pdf", f"_page_{page['metadata']['page']}.md") + pathlib.Path(outname).write_bytes(page["text"].encode()) + + t1 = time.perf_counter() + print(f"Markdown creation time for {converter.doc.name=} {round(t1 - t0, 2)} sec.") diff --git a/src/backend/semantic/test_pypdf_4.py b/src/backend/semantic/test_pypdf_4.py new file mode 100644 index 00000000..023139d9 --- /dev/null +++ b/src/backend/semantic/test_pypdf_4.py @@ -0,0 +1,356 @@ +import os +import pymupdf +from pymupdf4llm.helpers.get_text_lines import get_raw_lines, is_white +from pymupdf4llm.helpers.multi_column import column_boxes +from typing import List, Tuple, Dict, Union, Any + +if pymupdf.pymupdf_version_tuple < (1, 24, 2): + raise NotImplementedError("PyMuPDF version 1.24.2 or later is needed.") + +bullet = ("- ", "* ", chr(0xF0A7), chr(0xF0B7), chr(0xB7), chr(8226), chr(9679)) +GRAPHICS_TEXT = "\n![%s](%s)\n" + + +class IdentifyHeaders: + def __init__(self, doc: Union[str, pymupdf.Document], pages: List[int] = None, body_limit: float = 12) -> None: + if isinstance(doc, pymupdf.Document): + mydoc = doc + else: + mydoc = pymupdf.open(doc) + + if pages is None: + pages = list(range(mydoc.page_count)) + + fontsizes: Dict[int, int] = {} + for pno in pages: + page = mydoc.load_page(pno) + blocks = page.get_text("dict", flags=pymupdf.TEXTFLAGS_TEXT)["blocks"] + for span in [s for b in blocks for l in b["lines"] for s in l["spans"] if not is_white(s["text"])]: + fontsz = round(span["size"]) + count = fontsizes.get(fontsz, 0) + len(span["text"].strip()) + fontsizes[fontsz] = count + + if mydoc != doc: + mydoc.close() + + self.header_id: Dict[int, str] = {} + temp = sorted([(k, v) for k, v in fontsizes.items()], key=lambda i: i[1], reverse=True) + if temp: + b_limit = max(body_limit, temp[0][0]) + else: + b_limit = body_limit + + sizes = sorted([f for f in fontsizes.keys() if f > b_limit], reverse=True)[:6] + + for i, size in enumerate(sizes): + self.header_id[size] = "#" * (i + 1) + " " + + def get_header_id(self, span: dict, page: pymupdf.Page = None) -> str: + fontsize = round(span["size"]) + return self.header_id.get(fontsize, "") + + +class MarkdownConverter: + def __init__(self, doc_path: str, pages: List[int] = None, body_limit: float = 12, write_images: bool = False, + page_chunks: bool = False, + margins: Union[float, Tuple[float, float], Tuple[float, float, float, float]] = ( + 0, 50, 0, 50)) -> None: + self.doc_path = doc_path + self.doc = pymupdf.open(doc_path) + self.pages = pages if pages is not None else list(range(self.doc.page_count)) + self.body_limit = body_limit + self.write_images = write_images + self.page_chunks = page_chunks + self.margins = self._validate_margins(margins) + self.hdr_info = IdentifyHeaders(self.doc, pages=self.pages, body_limit=body_limit) + + def _validate_margins(self, margins: Union[float, Tuple[float, float], Tuple[float, float, float, float]]) -> Tuple[ + float, float, float, float]: + if hasattr(margins, "__float__"): + return (margins, margins, margins, margins) + if len(margins) == 2: + return (0, margins[0], 0, margins[1]) + if len(margins) != 4: + raise ValueError("Margins must have length 2 or 4 or be a number.") + if not all([hasattr(m, "__float__") for m in margins]): + raise ValueError("Margin values must be numbers") + return tuple(margins) + + def convert_to_markdown(self) -> Union[str, List[Dict[str, Any]]]: + document_output: Union[str, List[Dict[str, Any]]] = "" if not self.page_chunks else [] + toc = self.doc.get_toc() + textflags = pymupdf.TEXT_DEHYPHENATE | pymupdf.TEXT_MEDIABOX_CLIP + + for pno in self.pages: + page_output, images, tables, graphics = self._get_page_output(pno, textflags) + if not self.page_chunks: + document_output += page_output + else: + page_tocs = [t for t in toc if t[-1] == pno + 1] + metadata = self._get_metadata(pno) + document_output.append({ + "metadata": metadata, + "toc_items": page_tocs, + "tables": tables, + "images": images, + "graphics": graphics, + "text": page_output, + }) + + return document_output + + def _get_metadata(self, pno: int) -> Dict[str, Union[str, int]]: + meta = self.doc.metadata.copy() + meta["file_path"] = self.doc.name + meta["page_count"] = self.doc.page_count + meta["page"] = pno + 1 + return meta + + def _get_page_output(self, pno: int, textflags: int) -> Tuple[ + str, List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]: + page = self.doc[pno] + md_string = "" + left, top, right, bottom = self.margins + clip = page.rect + (left, top, -right, -bottom) + links = [l for l in page.get_links() if l["kind"] == 2] + textpage = page.get_textpage(flags=textflags, clip=clip) + img_info = [img for img in page.get_image_info() if img["bbox"] in clip] + images = img_info[:] + tables = [] + graphics = [] + tabs = page.find_tables(clip=clip, strategy="lines_strict") + tab_rects = self._get_tab_rects(tabs) + tab_rects0 = list(tab_rects.values()) + paths, vg_clusters0 = self._get_paths_and_graphics(page, tab_rects0, clip, img_info) + vg_clusters = {i: r for i, r in enumerate(vg_clusters0)} + text_rects = column_boxes(page, paths=paths, no_image_text=self.write_images, textpage=textpage, + avoid=tab_rects0 + vg_clusters0) + + for text_rect in text_rects: + md_string += self._output_tables(tabs, text_rect, tab_rects) + md_string += self._output_images(page, text_rect, vg_clusters) + md_string += self._write_text(page, textpage, text_rect, tabs, tab_rects, vg_clusters, links) + + md_string += self._output_tables(tabs, None, tab_rects) + md_string += self._output_images(page, None, vg_clusters) + md_string += "\n-----\n\n" + while md_string.startswith("\n"): + md_string = md_string[1:] + + return md_string, images, tables, graphics + + def _get_tab_rects(self, tabs: List[Any]) -> Dict[int, pymupdf.Rect]: + tab_rects: Dict[int, pymupdf.Rect] = {} + for i, t in enumerate(tabs): + tab_rects[i] = pymupdf.Rect(t.bbox) | pymupdf.Rect(t.header.bbox) + return tab_rects + + def _get_paths_and_graphics(self, page: pymupdf.Page, tab_rects0: List[pymupdf.Rect], clip: pymupdf.Rect, + img_info: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[pymupdf.Rect]]: + page_clip = page.rect + (36, 36, -36, -36) + paths = [p for p in page.get_drawings() if + not self._intersects_rects(p["rect"], tab_rects0) and p["rect"] in page_clip and p[ + "rect"].width < page_clip.width and p["rect"].height < page_clip.height] + vg_clusters = [r for r in page.cluster_drawings(drawings=paths) if + not self._intersects_rects(r, tab_rects0) and r.height > 20] + + if self.write_images: + vg_clusters += [pymupdf.Rect(i["bbox"]) for i in img_info] + + return paths, vg_clusters + + def _intersects_rects(self, rect: pymupdf.Rect, rect_list: List[pymupdf.Rect]) -> bool: + for r in rect_list: + if (rect.tl + rect.br) / 2 in r: + return True + return False + + def _output_tables(self, tabs: List[Any], text_rect: pymupdf.Rect, tab_rects: Dict[int, pymupdf.Rect]) -> str: + this_md = "" + if text_rect is not None: + for i, trect in sorted([(j[0], j[1]) for j in tab_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0)): + this_md += tabs[i].to_markdown(clean=False) + del tab_rects[i] + else: + for i, trect in sorted(tab_rects.items(), key=lambda j: (j[1].y1, j[1].x0)): + this_md += tabs[i].to_markdown(clean=False) + del tab_rects[i] + return this_md + + def _output_images(self, page: pymupdf.Page, text_rect: pymupdf.Rect, img_rects: Dict[int, pymupdf.Rect]) -> str: + if img_rects is None: + return "" + this_md = "" + if text_rect is not None: + for i, img_rect in sorted([(j[0], j[1]) for j in img_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0)): + pathname = self._save_image(page, img_rect, i) + if pathname: + this_md += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + else: + for i, img_rect in sorted(img_rects.items(), key=lambda j: (j[1].y1, j[1].x0)): + pathname = self._save_image(page, img_rect, i) + if pathname: + this_md += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + return this_md + + def _save_image(self, page: pymupdf.Page, rect: pymupdf.Rect, i: int) -> str: + filename = page.parent.name.replace("\\", "/") + image_path = f"{filename}-{page.number}-{i}.png" + if self.write_images: + pix = page.get_pixmap(clip=rect) + pix.save(image_path) + del pix + return os.path.basename(image_path) + return "" + + def _write_text(self, page: pymupdf.Page, textpage: pymupdf.TextPage, clip: pymupdf.Rect, tabs: List[Any], + tab_rects: Dict[int, pymupdf.Rect], img_rects: Dict[int, pymupdf.Rect], + links: List[Dict[str, Any]]) -> str: + out_string = "" + nlines = get_raw_lines(textpage, clip=clip, tolerance=3) + tab_rects0 = list(tab_rects.values()) + img_rects0 = list(img_rects.values()) + prev_lrect = None + prev_bno = -1 + code = False + prev_hdr_string = None + + for lrect, spans in nlines: + if self._intersects_rects(lrect, tab_rects0) or self._intersects_rects(lrect, img_rects0): + continue + + for i, tab_rect in sorted( + [j for j in tab_rects.items() if j[1].y1 <= lrect.y0 and not (j[1] & clip).is_empty], + key=lambda j: (j[1].y1, j[1].x0)): + out_string += "\n" + tabs[i].to_markdown(clean=False) + "\n" + del tab_rects[i] + + for i, img_rect in sorted( + [j for j in img_rects.items() if j[1].y1 <= lrect.y0 and not (j[1] & clip).is_empty], + key=lambda j: (j[1].y1, j[1].x0)): + pathname = self._save_image(page, img_rect, i) + if pathname: + out_string += GRAPHICS_TEXT % (pathname, pathname) + del img_rects[i] + + text = " ".join([s["text"] for s in spans]) + all_mono = all([s["flags"] & 8 for s in spans]) + + if all_mono: + if not code: + out_string += "```\n" + code = True + delta = int((lrect.x0 - clip.x0) / (spans[0]["size"] * 0.5)) + indent = " " * delta + out_string += indent + text + "\n" + continue + + span0 = spans[0] + bno = span0["block"] + if bno != prev_bno: + out_string += "\n" + prev_bno = bno + + if (prev_lrect and lrect.y1 - prev_lrect.y1 > lrect.height * 1.5 or span0["text"].startswith("[") or span0[ + "text"].startswith(bullet) or span0["flags"] & 1): + out_string += "\n" + prev_lrect = lrect + + hdr_string = self.hdr_info.get_header_id(span0, page=page) + if hdr_string and hdr_string == prev_hdr_string: + out_string = out_string[:-1] + " " + text + "\n" + continue + + prev_hdr_string = hdr_string + if hdr_string.startswith("#"): + out_string += hdr_string + text + "\n" + continue + + if code: + out_string += "```\n" + code = False + + for i, s in enumerate(spans): + mono = s["flags"] & 8 + bold = s["flags"] & 16 + italic = s["flags"] & 2 + + if mono: + out_string += f"`{s['text'].strip()}` " + else: + prefix = "" + suffix = "" + if hdr_string == "": + if bold: + prefix = "**" + suffix += "**" + if italic: + prefix += "_" + suffix = "_" + suffix + + ltext = self._resolve_links(links, s) + if ltext: + text = f"{hdr_string}{prefix}{ltext}{suffix} " + else: + text = f"{hdr_string}{prefix}{s['text'].strip()}{suffix} " + + if text.startswith(bullet): + text = "- " + text[1:] + out_string += text + + if not code: + out_string += "\n" + out_string += "\n" + if code: + out_string += "```\n" + code = False + + return out_string.replace(" \n", "\n").replace(" ", " ").replace("\n\n\n", "\n\n") + + def _resolve_links(self, links: List[Dict[str, Any]], span: Dict[str, Any]) -> Union[str, None]: + bbox = pymupdf.Rect(span["bbox"]) + bbox_area = 0.7 * abs(bbox) + for link in links: + hot = link["from"] + if abs(hot & bbox) >= bbox_area: + return f'[{span["text"].strip()}]({link["uri"]})' + return None + + +if __name__ == "__main__": + import pathlib + import sys + import time + + try: + filename = sys.argv[1] + except IndexError: + print(f"Usage:\npython {os.path.basename(__file__)} input.pdf") + sys.exit() + + t0 = time.perf_counter() + import pymupdf4llm + + md_text = pymupdf4llm.to_markdown(filename) + print(md_text) + + + # converter = MarkdownConverter(filename) + # md_string = converter.convert_to_markdown() + # + # if isinstance(md_string, str): + # print(md_string) + # outname = converter.doc_path.replace(".pdf", ".md") + # pathlib.Path(outname).write_bytes(md_string.encode()) + # else: + # for page in md_string: + # print(page["text"]) + # outname = converter.doc_path.replace(".pdf", f"_page_{page['metadata']['page']}.md") + # pathlib.Path(outname).write_bytes(page["text"].encode()) + + t1 = time.perf_counter() + print(f"Markdown creation time for {filename=} {round(t1 - t0, 2)} sec.") diff --git a/src/backend/semantic/test_whishper.py b/src/backend/semantic/test_whishper.py new file mode 100644 index 00000000..41272d28 --- /dev/null +++ b/src/backend/semantic/test_whishper.py @@ -0,0 +1,87 @@ +import os +import sys +import subprocess +import whisper +import torch + + +def extract_audio(filename): + # Generate the audio file name based on the input file name + audio_filename = filename.rsplit('.', 1)[0] + '_extracted.wav' + + # Check if the input file is already a .wav file + if filename.endswith('.wav'): + return filename + + # Check if the audio file already exists and delete it if it does + if os.path.exists(audio_filename): + os.remove(audio_filename) + + # Use FFmpeg to extract the audio + subprocess.run(['ffmpeg', '-i', filename, '-q:a', '0', '-map', 'a', audio_filename], check=True) + + return audio_filename + +def transcribe_audio(audio_filename): + # Detect the available device + device = "cpu" + if torch.cuda.is_available(): + device = "cuda" + elif torch.backends.mps.is_available() and torch.backends.mps.is_built(): + device = "mps" + + print(f"Using device: {device}") + + try: + # Load the Whisper model on the detected device + model = whisper.load_model("base", device=device).to(device) + + # Transcribe the audio file + result = model.transcribe(audio_filename) + except Exception as e: + print(f"Encountered an error with device {device}: {e}") + if device != "cpu": + print("Falling back to CPU.") + device = "cpu" + model = whisper.load_model("base", device=device) + result = model.transcribe(audio_filename) + else: + raise e + + return result["text"] + +def extract_text_media(filename): + # Extract audio from the media file + audio_filename = extract_audio(filename) + + # Transcribe the extracted audio + transcription = transcribe_audio(audio_filename) + + return transcription + +if __name__ == "__main__": + import pathlib + + try: + filename = sys.argv[1] + except IndexError: + print(f"Usage:\npython {os.path.basename(__file__)} input_file") + sys.exit() + + # Ensure the file exists + if not os.path.exists(filename): + print(f"File not found: {filename}") + sys.exit() + + # Supported file extensions + supported_extensions = {'mp4', 'mp3', 'mpeg', 'mpga', 'm4a', 'wav', 'webm', 'mov'} + + # Check if the file has a supported extension + file_extension = pathlib.Path(filename).suffix[1:].lower() + if file_extension not in supported_extensions: + print(f"Unsupported file format: {file_extension}") + print(f"Supported formats are: {', '.join(supported_extensions)}") + sys.exit() + + # Extract and print the text from the media file + print(extract_text_media(filename)) diff --git a/src/backend/semantic/test_youtube_transcript.py b/src/backend/semantic/test_youtube_transcript.py new file mode 100644 index 00000000..43313a5b --- /dev/null +++ b/src/backend/semantic/test_youtube_transcript.py @@ -0,0 +1,36 @@ +import asyncio +import datetime +import logging +import os +import time + +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.gen_types.file_type_pb2 import FileType +from lib.semantic.semantic_factory import SemanticFactory + + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) +logger = logging.getLogger(__name__) + + +async def main(): + semantic_data = SemanticData() + semantic_data.document_id = 123 + semantic_data.url = ("https://www.youtube.com/watch?v=UbDyjIIGaxQ") + semantic_data.file_type = FileType.YT + + # performing semantic analysis on the source + semantic = SemanticFactory.create_semantic_analyzer(semantic_data.file_type) + + entities_analyzed = semantic.analyze(data=semantic_data, full_process_start_time=time.time(), + ack_wait=30, cockroach_url="") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/backend/semantic/todo.md b/src/backend/semantic/todo.md new file mode 100644 index 00000000..539fcbfa --- /dev/null +++ b/src/backend/semantic/todo.md @@ -0,0 +1 @@ +testare quando si super ack_wai \ No newline at end of file diff --git a/src/backend/temp/ BWI .docx b/src/backend/temp/ BWI .docx new file mode 100644 index 00000000..0e21b5de Binary files /dev/null and b/src/backend/temp/ BWI .docx differ diff --git a/src/backend/temp/ Collaboard EKS cluster.docx b/src/backend/temp/ Collaboard EKS cluster.docx new file mode 100644 index 00000000..c0dc4aa3 Binary files /dev/null and b/src/backend/temp/ Collaboard EKS cluster.docx differ diff --git a/src/backend/temp/ Collaboard Sizing Recommendations.docx b/src/backend/temp/ Collaboard Sizing Recommendations.docx new file mode 100644 index 00000000..83cb8dea Binary files /dev/null and b/src/backend/temp/ Collaboard Sizing Recommendations.docx differ diff --git a/src/backend/temp/ Collaboard on Kubernetes.docx b/src/backend/temp/ Collaboard on Kubernetes.docx new file mode 100644 index 00000000..89bc9d36 Binary files /dev/null and b/src/backend/temp/ Collaboard on Kubernetes.docx differ diff --git a/src/backend/temp/ Collaboard.docx b/src/backend/temp/ Collaboard.docx new file mode 100644 index 00000000..f750f2da Binary files /dev/null and b/src/backend/temp/ Collaboard.docx differ diff --git a/src/backend/temp/ Database Details.docx b/src/backend/temp/ Database Details.docx new file mode 100644 index 00000000..1c822c67 Binary files /dev/null and b/src/backend/temp/ Database Details.docx differ diff --git a/src/backend/temp/ Database Overview.docx b/src/backend/temp/ Database Overview.docx new file mode 100644 index 00000000..7ac011b8 Binary files /dev/null and b/src/backend/temp/ Database Overview.docx differ diff --git a/src/backend/temp/ Dev and Release guidelines.docx b/src/backend/temp/ Dev and Release guidelines.docx new file mode 100644 index 00000000..a06cc247 Binary files /dev/null and b/src/backend/temp/ Dev and Release guidelines.docx differ diff --git a/src/backend/temp/ Installing Collaboard on Containers.pdf b/src/backend/temp/ Installing Collaboard on Containers.pdf new file mode 100644 index 00000000..839fa141 Binary files /dev/null and b/src/backend/temp/ Installing Collaboard on Containers.pdf differ diff --git a/src/backend/temp/ Installing Collaboard on Docker.docx b/src/backend/temp/ Installing Collaboard on Docker.docx new file mode 100644 index 00000000..0fa483d8 Binary files /dev/null and b/src/backend/temp/ Installing Collaboard on Docker.docx differ diff --git a/src/backend/temp/ Load Test Collaboard .docx b/src/backend/temp/ Load Test Collaboard .docx new file mode 100644 index 00000000..6639da40 Binary files /dev/null and b/src/backend/temp/ Load Test Collaboard .docx differ diff --git a/src/backend/temp/ NDA.docx b/src/backend/temp/ NDA.docx new file mode 100644 index 00000000..9afab88d Binary files /dev/null and b/src/backend/temp/ NDA.docx differ diff --git a/src/backend/temp/ Overview.docx b/src/backend/temp/ Overview.docx new file mode 100644 index 00000000..c3599350 Binary files /dev/null and b/src/backend/temp/ Overview.docx differ diff --git a/src/backend/temp/0252dc29202e4e168f2d569907d9d5cd_.md b/src/backend/temp/0252dc29202e4e168f2d569907d9d5cd_.md new file mode 100644 index 00000000..7b5bd54e --- /dev/null +++ b/src/backend/temp/0252dc29202e4e168f2d569907d9d5cd_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/03cce1b5fe7748309a702a98eac2ff00_.md b/src/backend/temp/03cce1b5fe7748309a702a98eac2ff00_.md new file mode 100644 index 00000000..81a3ce1d --- /dev/null +++ b/src/backend/temp/03cce1b5fe7748309a702a98eac2ff00_.md @@ -0,0 +1,63 @@ +Gian Paolo Santopaolo +```html +
+
+
Corrado Cavalli we are waiting for your estimations about:
+ +
 
+ + +
+ +

Time to port DocumentExplorer on Xamarin Forms:

+ + + +

 

+ + + +

Time to port CollaBoard on Xamarin Forms:

+ + +
 
+ +
 
+
+
+
+
+``` + +Gian Paolo Santopaolo +```html +
+
Corrado Cavalli the estimate you forwarded has to be done again because some of the part have already be ported. + +

Please do a new estimate based on the current status  for both  CB and DE

+ + + +

 I've added a page to the dev note document as well..

+ + +
thanks
+ + +

thanks

+ + + +

/gp

+ +
+
+``` + +Corrado Cavalli +```html +
Quick Update: DocumentExplorer now compiles fine after migration works, but I need to do a complete test to check if all modules are working good like before, hope to complete all test tomorrow morning.
+ +
Regarding estimation I will forward the mail sent you previously regarding the estimation
+
+``` diff --git a/src/backend/temp/073bd5a059184829b4f9390730c766a1_.md b/src/backend/temp/073bd5a059184829b4f9390730c766a1_.md new file mode 100644 index 00000000..7359b8a8 --- /dev/null +++ b/src/backend/temp/073bd5a059184829b4f9390730c766a1_.md @@ -0,0 +1,4 @@ +Gian Paolo Santopaolo +```html + +``` diff --git a/src/backend/temp/0e905a2df2e54a93b675ca00d31579f8_.md b/src/backend/temp/0e905a2df2e54a93b675ca00d31579f8_.md new file mode 100644 index 00000000..71c0e950 --- /dev/null +++ b/src/backend/temp/0e905a2df2e54a93b675ca00d31579f8_.md @@ -0,0 +1,4 @@ +Corrado Cavalli +```html +As for CollaBoard, also DE has been refactored to use the new libraries, the build is named RS2-Prism, give it a try to check that no regressions have been introduced by the refactoring. +``` diff --git a/src/backend/temp/112283c1443b4e259859f519306507fe_.md b/src/backend/temp/112283c1443b4e259859f519306507fe_.md new file mode 100644 index 00000000..f73f01da --- /dev/null +++ b/src/backend/temp/112283c1443b4e259859f519306507fe_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/16883b189a6346be86a53fe96c387342_.md b/src/backend/temp/16883b189a6346be86a53fe96c387342_.md new file mode 100644 index 00000000..584db446 --- /dev/null +++ b/src/backend/temp/16883b189a6346be86a53fe96c387342_.md @@ -0,0 +1,13 @@ +Corrado Cavalli +```html +
+
+
I've created a build of CB (RS2-PRISM) nothing new compared to dev/vnext but it has been refactored to be ported to netstandard and supports latest 1809 build of Windows. It still targets RS2 for compatibility. Please start using it and report any issue you find with it.
+ +
please uninstall any previous version before installing this build.
+ +
Thanks.
+
+
+
+``` diff --git a/src/backend/temp/172dd8fe6c9e458e8be551e0d6980016_.md b/src/backend/temp/172dd8fe6c9e458e8be551e0d6980016_.md new file mode 100644 index 00000000..0e04b78c --- /dev/null +++ b/src/backend/temp/172dd8fe6c9e458e8be551e0d6980016_.md @@ -0,0 +1,5 @@ +Gian Paolo Santopaolo +```html +
Corrado Cavalli I've updated DevNotes (Xamarin Estimates). Please add the corrisponding wor items into VSTS and add the links on DevNots so that Andreas Görög can have an overview of the overall work status
+
+``` diff --git a/src/backend/temp/186a1ed39f334d4cbf8c5338ced2e7dd_.md b/src/backend/temp/186a1ed39f334d4cbf8c5338ced2e7dd_.md new file mode 100644 index 00000000..09aac838 --- /dev/null +++ b/src/backend/temp/186a1ed39f334d4cbf8c5338ced2e7dd_.md @@ -0,0 +1,9 @@ +Gian Paolo Santopaolo +```html +
+
+
Corrado Cavalli tried to ping you both on skype (classic) and on Team's chat, no answer. Need to talk to you, please ping me back. Thank you!
+
+
+
+``` diff --git a/src/backend/temp/1Template.docx b/src/backend/temp/1Template.docx new file mode 100644 index 00000000..2b30c765 Binary files /dev/null and b/src/backend/temp/1Template.docx differ diff --git a/src/backend/temp/1a99d3f434ac49838c766d4eb1d86796_.md b/src/backend/temp/1a99d3f434ac49838c766d4eb1d86796_.md new file mode 100644 index 00000000..bf4095a1 --- /dev/null +++ b/src/backend/temp/1a99d3f434ac49838c766d4eb1d86796_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/1f8ed7c04e2e4be9b7fca9add36037cf_.md b/src/backend/temp/1f8ed7c04e2e4be9b7fca9add36037cf_.md new file mode 100644 index 00000000..8e8e6eec --- /dev/null +++ b/src/backend/temp/1f8ed7c04e2e4be9b7fca9add36037cf_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/20180226 CollaBoard Architecture.docx b/src/backend/temp/20180226 CollaBoard Architecture.docx new file mode 100644 index 00000000..ab1eb2f9 Binary files /dev/null and b/src/backend/temp/20180226 CollaBoard Architecture.docx differ diff --git a/src/backend/temp/20210129 Deploying Scalable Collaboard on Docker.docx b/src/backend/temp/20210129 Deploying Scalable Collaboard on Docker.docx new file mode 100644 index 00000000..e018c959 Binary files /dev/null and b/src/backend/temp/20210129 Deploying Scalable Collaboard on Docker.docx differ diff --git a/src/backend/temp/20210708.pdf b/src/backend/temp/20210708.pdf new file mode 100644 index 00000000..f60765b7 Binary files /dev/null and b/src/backend/temp/20210708.pdf differ diff --git a/src/backend/temp/2154618ddb0a42268d512dd45921065e_.md b/src/backend/temp/2154618ddb0a42268d512dd45921065e_.md new file mode 100644 index 00000000..04cbd83c --- /dev/null +++ b/src/backend/temp/2154618ddb0a42268d512dd45921065e_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/22e51cdea9d34e1c8f5fd9d7d534b648_.md b/src/backend/temp/22e51cdea9d34e1c8f5fd9d7d534b648_.md new file mode 100644 index 00000000..8d747ca5 --- /dev/null +++ b/src/backend/temp/22e51cdea9d34e1c8f5fd9d7d534b648_.md @@ -0,0 +1,22 @@ +Michael Görög +```html +
Dev Team: It seems there is a problem with CollaBoard. When starting the app we (and partners and customers) get this error messages:
+
+``` + +Michael Görög +```html +Now it is working again. +``` + +Michael Görög +```html +
+
+
 
+
+ +
 
+
+
+``` diff --git a/src/backend/temp/245620f8087a4367a2e177c657409dfc_.md b/src/backend/temp/245620f8087a4367a2e177c657409dfc_.md new file mode 100644 index 00000000..52d79523 --- /dev/null +++ b/src/backend/temp/245620f8087a4367a2e177c657409dfc_.md @@ -0,0 +1,4 @@ +Corrado Cavalli +```html +I've created the DevOps tasks and added links into DevNotes +``` diff --git a/src/backend/temp/2888cb6dd10a429098689b267d491e50_.md b/src/backend/temp/2888cb6dd10a429098689b267d491e50_.md new file mode 100644 index 00000000..584db446 --- /dev/null +++ b/src/backend/temp/2888cb6dd10a429098689b267d491e50_.md @@ -0,0 +1,13 @@ +Corrado Cavalli +```html +
+
+
I've created a build of CB (RS2-PRISM) nothing new compared to dev/vnext but it has been refactored to be ported to netstandard and supports latest 1809 build of Windows. It still targets RS2 for compatibility. Please start using it and report any issue you find with it.
+ +
please uninstall any previous version before installing this build.
+ +
Thanks.
+
+
+
+``` diff --git a/src/backend/temp/2db8b8afd25e4ee1b9a5b1bd4a702884_.md b/src/backend/temp/2db8b8afd25e4ee1b9a5b1bd4a702884_.md new file mode 100644 index 00000000..5ed47406 --- /dev/null +++ b/src/backend/temp/2db8b8afd25e4ee1b9a5b1bd4a702884_.md @@ -0,0 +1,70 @@ +Michael Görög +```html +
Corrado Cavalli: I downloaded this build and tested if I can login to the IBV environment. +
+
 
+
+ +
But I get the error message, that the username is not valid. 
+
+
+``` + +Michael Görög +```html +
Corrado Cavalli: Thank you. Now I can login, but not connect to the server. I will check with Oli if Server and Storage address are correct.
+
+``` + +Corrado Cavalli +```html +
Michael Görög Ah... so that is not the right build... you should have received one yesterday evening with IBV ON PREMISE
+
+``` + +Michael Görög +```html +
Corrado Cavalli: Yes this is working, but we need to test it internaly to check if DFS is working or not. So we need a version where we are able to connect to Z. 
+
+``` + +Corrado Cavalli +```html +
Michael Görög does it work with Guest1/... user?
+
+``` + +Michael Görög +```html +
Corrado Cavalli: I installed 1.3.61. If I click on AD I still see the Guest user. When I try to connect using my AD credentials, I still get the same error: +
+
 
+
+ +
 
+
+
+``` + +Corrado Cavalli +```html +
Michael Görög Let me schedule a new build (61) and let me know if it works with that...
+
+``` + +Michael Görög +```html +
Corrado Cavalli: Yes I did. I tested on another device as well (uninstall and reinstall) but I have the issue that I cant login. What other options do we have? (Just for your Info, when I click on AD Login, User Name, Passwort and Domain are already filled with the Guest1 User).
+
+``` + +Corrado Cavalli +```html +
Michael Görög build is right, that commit refer to IBV on premise server, did you try uninstall previous version?
+
+``` + +Michael Görög +```html +Can you please check if this is the right build or not? +``` diff --git a/src/backend/temp/3559f5e861254686854dc254a3e6de68_.md b/src/backend/temp/3559f5e861254686854dc254a3e6de68_.md new file mode 100644 index 00000000..8f7ccb0a --- /dev/null +++ b/src/backend/temp/3559f5e861254686854dc254a3e6de68_.md @@ -0,0 +1,5 @@ +Gian Paolo Santopaolo +```html +
Dev Team FYI there is a bug on CB on premises, since several weeks, that will prevent it to start-up (all on-premises installations are affected). We are fixing it and will publish a new version for you to test. Corrado Cavalli will inform you when ready
+
+``` diff --git a/src/backend/temp/36f5ca62049547e4917947d20ec40635_.md b/src/backend/temp/36f5ca62049547e4917947d20ec40635_.md new file mode 100644 index 00000000..0e0b3a2e --- /dev/null +++ b/src/backend/temp/36f5ca62049547e4917947d20ec40635_.md @@ -0,0 +1,16 @@ +Michael Görög +```html +
Corrado Cavalli: Hi Corrado, is this the new client for Midor with the adjustments? +
+
 
+
+ +
If yes I would like to send it to Midor today. 
+
+
+``` + +Corrado Cavalli +```html +Yes it is, I also send it to the store. +``` diff --git a/src/backend/temp/3e7307779fc44782b7ae4f70868f443e_.md b/src/backend/temp/3e7307779fc44782b7ae4f70868f443e_.md new file mode 100644 index 00000000..bcc082a2 --- /dev/null +++ b/src/backend/temp/3e7307779fc44782b7ae4f70868f443e_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/4b87fd186cc1488cb06a9f90a8390a7f_.md b/src/backend/temp/4b87fd186cc1488cb06a9f90a8390a7f_.md new file mode 100644 index 00000000..0e0b3a2e --- /dev/null +++ b/src/backend/temp/4b87fd186cc1488cb06a9f90a8390a7f_.md @@ -0,0 +1,16 @@ +Michael Görög +```html +
Corrado Cavalli: Hi Corrado, is this the new client for Midor with the adjustments? +
+
 
+
+ +
If yes I would like to send it to Midor today. 
+
+
+``` + +Corrado Cavalli +```html +Yes it is, I also send it to the store. +``` diff --git a/src/backend/temp/4e453eae76064a9083e1fb6b8ada32aa_.md b/src/backend/temp/4e453eae76064a9083e1fb6b8ada32aa_.md new file mode 100644 index 00000000..4e003e4c --- /dev/null +++ b/src/backend/temp/4e453eae76064a9083e1fb6b8ada32aa_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/4f986544cafe4682be029a1d1e0d1e49_.md b/src/backend/temp/4f986544cafe4682be029a1d1e0d1e49_.md new file mode 100644 index 00000000..d55c8d1c --- /dev/null +++ b/src/backend/temp/4f986544cafe4682be029a1d1e0d1e49_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/52cfadbb7e504f2b95e21d8b50685542_.md b/src/backend/temp/52cfadbb7e504f2b95e21d8b50685542_.md new file mode 100644 index 00000000..c6d0974a --- /dev/null +++ b/src/backend/temp/52cfadbb7e504f2b95e21d8b50685542_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/73c3cec7bdcf4240af0d36faeea82a83_.md b/src/backend/temp/73c3cec7bdcf4240af0d36faeea82a83_.md new file mode 100644 index 00000000..71c0e950 --- /dev/null +++ b/src/backend/temp/73c3cec7bdcf4240af0d36faeea82a83_.md @@ -0,0 +1,4 @@ +Corrado Cavalli +```html +As for CollaBoard, also DE has been refactored to use the new libraries, the build is named RS2-Prism, give it a try to check that no regressions have been introduced by the refactoring. +``` diff --git a/src/backend/temp/75c0e26d55ac43e1a0f0ceecba891176_.md b/src/backend/temp/75c0e26d55ac43e1a0f0ceecba891176_.md new file mode 100644 index 00000000..49389afa --- /dev/null +++ b/src/backend/temp/75c0e26d55ac43e1a0f0ceecba891176_.md @@ -0,0 +1,17 @@ +Corrado Cavalli +```html +
+
Dev Team We have fixed CB On Premise (Azure), would be nice if we could test the installation on IBV Server, how can we proceed?
+
+``` + +Andreas Görög +```html +Let me know how we can pull the newest server and client versions. Actually we still have the version before the porting in the local sln's and have not pulled anything since then.  +``` + +Gian Paolo Santopaolo +```html +
I'm preparing for the IT team all the server side binaries. Andreas Görög then it's on you. Let us know how to proceed
+
+``` diff --git a/src/backend/temp/76aab435456a48c5906aadb8f9be8e8d_.md b/src/backend/temp/76aab435456a48c5906aadb8f9be8e8d_.md new file mode 100644 index 00000000..8d747ca5 --- /dev/null +++ b/src/backend/temp/76aab435456a48c5906aadb8f9be8e8d_.md @@ -0,0 +1,22 @@ +Michael Görög +```html +
Dev Team: It seems there is a problem with CollaBoard. When starting the app we (and partners and customers) get this error messages:
+
+``` + +Michael Görög +```html +Now it is working again. +``` + +Michael Görög +```html +
+
+
 
+
+ +
 
+
+
+``` diff --git a/src/backend/temp/76ff0f9c328943ac8c77d9ecc1ec803f_.md b/src/backend/temp/76ff0f9c328943ac8c77d9ecc1ec803f_.md new file mode 100644 index 00000000..e42b3c0d --- /dev/null +++ b/src/backend/temp/76ff0f9c328943ac8c77d9ecc1ec803f_.md @@ -0,0 +1,5 @@ +Andreas Görög +```html +
Corrado Cavalli: I got email at 11:10: [Build succeeded] DE-RS5-DEBUG-PROD - Document Explorer:dev/vnext - Document Explorer - a23f6066, but no email for CB
+
+``` diff --git a/src/backend/temp/7fc2f9add69a4a7dafe4f06b5882ed59_.md b/src/backend/temp/7fc2f9add69a4a7dafe4f06b5882ed59_.md new file mode 100644 index 00000000..c19d97f4 --- /dev/null +++ b/src/backend/temp/7fc2f9add69a4a7dafe4f06b5882ed59_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/8ad2bf82e4ff4ab8b626825e9209cb64_.md b/src/backend/temp/8ad2bf82e4ff4ab8b626825e9209cb64_.md new file mode 100644 index 00000000..89e9ac42 --- /dev/null +++ b/src/backend/temp/8ad2bf82e4ff4ab8b626825e9209cb64_.md @@ -0,0 +1,5 @@ +Gian Paolo Santopaolo +```html +
Andreas Görög Monday we need to deciede a few things togeter. We need to met. Send me an invite anytime your convenience, please
+
+``` diff --git a/src/backend/temp/8edcc647af9844688315f0868883ed2a_.md b/src/backend/temp/8edcc647af9844688315f0868883ed2a_.md new file mode 100644 index 00000000..8d747ca5 --- /dev/null +++ b/src/backend/temp/8edcc647af9844688315f0868883ed2a_.md @@ -0,0 +1,22 @@ +Michael Görög +```html +
Dev Team: It seems there is a problem with CollaBoard. When starting the app we (and partners and customers) get this error messages:
+
+``` + +Michael Görög +```html +Now it is working again. +``` + +Michael Görög +```html +
+
+
 
+
+ +
 
+
+
+``` diff --git a/src/backend/temp/8f0ca26f5d21490ab9f95f3db8e3c1ea_.md b/src/backend/temp/8f0ca26f5d21490ab9f95f3db8e3c1ea_.md new file mode 100644 index 00000000..f1c964bf --- /dev/null +++ b/src/backend/temp/8f0ca26f5d21490ab9f95f3db8e3c1ea_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/AzureExceptions.xlsx b/src/backend/temp/AzureExceptions.xlsx new file mode 100644 index 00000000..58bc9466 Binary files /dev/null and b/src/backend/temp/AzureExceptions.xlsx differ diff --git a/src/backend/temp/Cockroach.xlsx b/src/backend/temp/Cockroach.xlsx new file mode 100644 index 00000000..35d9de47 Binary files /dev/null and b/src/backend/temp/Cockroach.xlsx differ diff --git a/src/backend/temp/Collaboard monetization.docx b/src/backend/temp/Collaboard monetization.docx new file mode 100644 index 00000000..50492c7e Binary files /dev/null and b/src/backend/temp/Collaboard monetization.docx differ diff --git a/src/backend/temp/Collaboard_Containers.pdf b/src/backend/temp/Collaboard_Containers.pdf new file mode 100644 index 00000000..839fa141 Binary files /dev/null and b/src/backend/temp/Collaboard_Containers.pdf differ diff --git a/src/backend/temp/Confidential_Collaboard.pptx b/src/backend/temp/Confidential_Collaboard.pptx new file mode 100644 index 00000000..b5a74a62 Binary files /dev/null and b/src/backend/temp/Confidential_Collaboard.pptx differ diff --git a/src/backend/temp/Dependency injection and inversion of control.docx b/src/backend/temp/Dependency injection and inversion of control.docx new file mode 100644 index 00000000..6651f5ae Binary files /dev/null and b/src/backend/temp/Dependency injection and inversion of control.docx differ diff --git a/src/backend/temp/HDS19 OnPrem.docx b/src/backend/temp/HDS19 OnPrem.docx new file mode 100644 index 00000000..e27a8ad7 Binary files /dev/null and b/src/backend/temp/HDS19 OnPrem.docx differ diff --git a/src/backend/temp/How to connect to the database in OTC cloud.docx b/src/backend/temp/How to connect to the database in OTC cloud.docx new file mode 100644 index 00000000..c7a8fd48 Binary files /dev/null and b/src/backend/temp/How to connect to the database in OTC cloud.docx differ diff --git a/src/backend/temp/How to connect to the database in OTC cloud.pdf b/src/backend/temp/How to connect to the database in OTC cloud.pdf new file mode 100644 index 00000000..ad3998cd Binary files /dev/null and b/src/backend/temp/How to connect to the database in OTC cloud.pdf differ diff --git a/src/backend/temp/How to publish a new CollaBoard release.docx b/src/backend/temp/How to publish a new CollaBoard release.docx new file mode 100644 index 00000000..c1aae0a9 Binary files /dev/null and b/src/backend/temp/How to publish a new CollaBoard release.docx differ diff --git a/src/backend/temp/How to restore DB from OTC to k8s in the LAB.docx b/src/backend/temp/How to restore DB from OTC to k8s in the LAB.docx new file mode 100644 index 00000000..c4f792ca Binary files /dev/null and b/src/backend/temp/How to restore DB from OTC to k8s in the LAB.docx differ diff --git a/src/backend/temp/IBVCodeSigningInfos.txt b/src/backend/temp/IBVCodeSigningInfos.txt new file mode 100644 index 00000000..99345b30 --- /dev/null +++ b/src/backend/temp/IBVCodeSigningInfos.txt @@ -0,0 +1,5 @@ +PW: IBVCodeSign + +SUP: 28324 + + diff --git a/src/backend/temp/IBV_Dokumentation2021.docx b/src/backend/temp/IBV_Dokumentation2021.docx new file mode 100644 index 00000000..adf571e6 Binary files /dev/null and b/src/backend/temp/IBV_Dokumentation2021.docx differ diff --git a/src/backend/temp/Licensing.pdf b/src/backend/temp/Licensing.pdf new file mode 100644 index 00000000..781dd6ce Binary files /dev/null and b/src/backend/temp/Licensing.pdf differ diff --git a/src/backend/temp/Load Test Collaboard .docx b/src/backend/temp/Load Test Collaboard .docx new file mode 100644 index 00000000..fb4befbb Binary files /dev/null and b/src/backend/temp/Load Test Collaboard .docx differ diff --git a/src/backend/temp/Migration to RS5.docx b/src/backend/temp/Migration to RS5.docx new file mode 100644 index 00000000..25bd3dda Binary files /dev/null and b/src/backend/temp/Migration to RS5.docx differ diff --git a/src/backend/temp/ReadMe.txt b/src/backend/temp/ReadMe.txt new file mode 100644 index 00000000..7f96e37c --- /dev/null +++ b/src/backend/temp/ReadMe.txt @@ -0,0 +1,21 @@ +How to start with Socket messagnger: + +SERVER +1. Copy "Server" folder to a server +2. Edit Server.exe.config file to set up correct ip and port: + + + + +3. Open port on a server for incoming TCP connections +4. Run Server.exe + +Clients +1. Copy "Client" folder to a client environment +2. Edit Client.exe.config file to set up correct ip and port - it should machh to the server configuration: + + + + +3. Run Client.exe + diff --git a/src/backend/temp/SQL Scripts.pdf b/src/backend/temp/SQL Scripts.pdf new file mode 100644 index 00000000..6310531e Binary files /dev/null and b/src/backend/temp/SQL Scripts.pdf differ diff --git a/src/backend/temp/Security Overview Collaboard.docx b/src/backend/temp/Security Overview Collaboard.docx new file mode 100644 index 00000000..c729efbb Binary files /dev/null and b/src/backend/temp/Security Overview Collaboard.docx differ diff --git a/src/backend/temp/Support Desk User manual.docx b/src/backend/temp/Support Desk User manual.docx new file mode 100644 index 00000000..88db254e Binary files /dev/null and b/src/backend/temp/Support Desk User manual.docx differ diff --git a/src/backend/temp/Support Desk User manual.pdf b/src/backend/temp/Support Desk User manual.pdf new file mode 100644 index 00000000..d212f7f8 Binary files /dev/null and b/src/backend/temp/Support Desk User manual.pdf differ diff --git a/src/backend/temp/VPN_Manual.docx b/src/backend/temp/VPN_Manual.docx new file mode 100644 index 00000000..c822cd47 Binary files /dev/null and b/src/backend/temp/VPN_Manual.docx differ diff --git a/src/backend/temp/VSTS Migration Report.xlsx b/src/backend/temp/VSTS Migration Report.xlsx new file mode 100644 index 00000000..d359733f Binary files /dev/null and b/src/backend/temp/VSTS Migration Report.xlsx differ diff --git a/src/backend/temp/Vitaliy's Activity Report.xlsx b/src/backend/temp/Vitaliy's Activity Report.xlsx new file mode 100644 index 00000000..f3fdc1f5 Binary files /dev/null and b/src/backend/temp/Vitaliy's Activity Report.xlsx differ diff --git a/src/backend/temp/Wiki Pages User manual.docx b/src/backend/temp/Wiki Pages User manual.docx new file mode 100644 index 00000000..c8b185b4 Binary files /dev/null and b/src/backend/temp/Wiki Pages User manual.docx differ diff --git a/src/backend/temp/Wiki Pages User manual.pdf b/src/backend/temp/Wiki Pages User manual.pdf new file mode 100644 index 00000000..f6d3ccd9 Binary files /dev/null and b/src/backend/temp/Wiki Pages User manual.pdf differ diff --git a/src/backend/temp/a6b2c7db3d1548f98eeb0c9241989ce4_.md b/src/backend/temp/a6b2c7db3d1548f98eeb0c9241989ce4_.md new file mode 100644 index 00000000..1c556d21 --- /dev/null +++ b/src/backend/temp/a6b2c7db3d1548f98eeb0c9241989ce4_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/a7ac2a65885741d28996a321f048e546_.md b/src/backend/temp/a7ac2a65885741d28996a321f048e546_.md new file mode 100644 index 00000000..c19d97f4 --- /dev/null +++ b/src/backend/temp/a7ac2a65885741d28996a321f048e546_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/a9148a092e6344a7bca4cc2f049eda6a_.md b/src/backend/temp/a9148a092e6344a7bca4cc2f049eda6a_.md new file mode 100644 index 00000000..0e0b3a2e --- /dev/null +++ b/src/backend/temp/a9148a092e6344a7bca4cc2f049eda6a_.md @@ -0,0 +1,16 @@ +Michael Görög +```html +
Corrado Cavalli: Hi Corrado, is this the new client for Midor with the adjustments? +
+
 
+
+ +
If yes I would like to send it to Midor today. 
+
+
+``` + +Corrado Cavalli +```html +Yes it is, I also send it to the store. +``` diff --git a/src/backend/temp/b26f23ecbf6949cd9c74d02fe8f80e85_.md b/src/backend/temp/b26f23ecbf6949cd9c74d02fe8f80e85_.md new file mode 100644 index 00000000..89e9ac42 --- /dev/null +++ b/src/backend/temp/b26f23ecbf6949cd9c74d02fe8f80e85_.md @@ -0,0 +1,5 @@ +Gian Paolo Santopaolo +```html +
Andreas Görög Monday we need to deciede a few things togeter. We need to met. Send me an invite anytime your convenience, please
+
+``` diff --git a/src/backend/temp/b2b35778f65b41a2a3b6a8155394ea3e_.md b/src/backend/temp/b2b35778f65b41a2a3b6a8155394ea3e_.md new file mode 100644 index 00000000..452357fd --- /dev/null +++ b/src/backend/temp/b2b35778f65b41a2a3b6a8155394ea3e_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/b449642d714345a3bb0aad575d816516_.md b/src/backend/temp/b449642d714345a3bb0aad575d816516_.md new file mode 100644 index 00000000..bea951cf --- /dev/null +++ b/src/backend/temp/b449642d714345a3bb0aad575d816516_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/b5b19439f6b84f3a8e023288e954dddc_.md b/src/backend/temp/b5b19439f6b84f3a8e023288e954dddc_.md new file mode 100644 index 00000000..81a3ce1d --- /dev/null +++ b/src/backend/temp/b5b19439f6b84f3a8e023288e954dddc_.md @@ -0,0 +1,63 @@ +Gian Paolo Santopaolo +```html +
+
+
Corrado Cavalli we are waiting for your estimations about:
+ +
 
+ + +
+ +

Time to port DocumentExplorer on Xamarin Forms:

+ + + +

 

+ + + +

Time to port CollaBoard on Xamarin Forms:

+ + +
 
+ +
 
+
+
+
+
+``` + +Gian Paolo Santopaolo +```html +
+
Corrado Cavalli the estimate you forwarded has to be done again because some of the part have already be ported. + +

Please do a new estimate based on the current status  for both  CB and DE

+ + + +

 I've added a page to the dev note document as well..

+ + +
thanks
+ + +

thanks

+ + + +

/gp

+ +
+
+``` + +Corrado Cavalli +```html +
Quick Update: DocumentExplorer now compiles fine after migration works, but I need to do a complete test to check if all modules are working good like before, hope to complete all test tomorrow morning.
+ +
Regarding estimation I will forward the mail sent you previously regarding the estimation
+
+``` diff --git a/src/backend/temp/b9f01acf7efc43c8a05bc1be2110a87e_.md b/src/backend/temp/b9f01acf7efc43c8a05bc1be2110a87e_.md new file mode 100644 index 00000000..4758c34b --- /dev/null +++ b/src/backend/temp/b9f01acf7efc43c8a05bc1be2110a87e_.md @@ -0,0 +1,39 @@ +Michael Görög +```html +
Gian Paolo Santopaolo: At the moment we have several bugs in DE that should be fixed now, but are Hub related so I did not test them yet. Is it ok if Corrado publishes a new build to the store, so we can test the Hub related issues this week? 
+
+``` + +Gian Paolo Santopaolo +```html + +``` + +Michael Görög +```html +
Corrado Cavalli: Thank you, I will test tomorrow the issues related to the hub. Gian Paolo Santopaolo: What do you think regarding the issue 218? The issue here is that DropBox will never work on the Hub, because it is not possible to copy&past the code, that needs to be copied to make it work. 
+
+``` + +Corrado Cavalli +```html +
Michael Görög Several? 212 was updated on Nov 30th, 218 We can't do nothing if Hub doesn't provide a copy/paste option, 216 should have been fixed two weeks ago. Others? Meantime I will prepare another build for the Hub.
+
+``` + +Michael Görög +```html +
Corrado Cavalli: For example the bug where on the Hub was the singel user interface when starting the app the first time (212) or the issue with DropBox (218) where it is not really possible to use it on the Hub, because the code can't be copied.  Or (216) where we got a Panda when doing copy & paste from google drive. I am not sure if these are really Hub related but we spotted these bugs, when doing the test on the Hub.
+
+``` + +Corrado Cavalli +```html +
Michael Görög What do you mean with Hub related?
+
+``` + +Gian Paolo Santopaolo +```html +
Anything that can help is ok. Corrado Cavalli how is possible bugs only on the Hub?
+``` diff --git a/src/backend/temp/bd2fbf0fb3ad4332a8421439d23e32dd_.md b/src/backend/temp/bd2fbf0fb3ad4332a8421439d23e32dd_.md new file mode 100644 index 00000000..e9f948e3 --- /dev/null +++ b/src/backend/temp/bd2fbf0fb3ad4332a8421439d23e32dd_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/befb1b5f2de7452cbd59eb82c242bd5b_.md b/src/backend/temp/befb1b5f2de7452cbd59eb82c242bd5b_.md new file mode 100644 index 00000000..8f7ccb0a --- /dev/null +++ b/src/backend/temp/befb1b5f2de7452cbd59eb82c242bd5b_.md @@ -0,0 +1,5 @@ +Gian Paolo Santopaolo +```html +
Dev Team FYI there is a bug on CB on premises, since several weeks, that will prevent it to start-up (all on-premises installations are affected). We are fixing it and will publish a new version for you to test. Corrado Cavalli will inform you when ready
+
+``` diff --git a/src/backend/temp/c205a86a830f40fa91b2c6dbb405c939_.md b/src/backend/temp/c205a86a830f40fa91b2c6dbb405c939_.md new file mode 100644 index 00000000..4758c34b --- /dev/null +++ b/src/backend/temp/c205a86a830f40fa91b2c6dbb405c939_.md @@ -0,0 +1,39 @@ +Michael Görög +```html +
Gian Paolo Santopaolo: At the moment we have several bugs in DE that should be fixed now, but are Hub related so I did not test them yet. Is it ok if Corrado publishes a new build to the store, so we can test the Hub related issues this week? 
+
+``` + +Gian Paolo Santopaolo +```html + +``` + +Michael Görög +```html +
Corrado Cavalli: Thank you, I will test tomorrow the issues related to the hub. Gian Paolo Santopaolo: What do you think regarding the issue 218? The issue here is that DropBox will never work on the Hub, because it is not possible to copy&past the code, that needs to be copied to make it work. 
+
+``` + +Corrado Cavalli +```html +
Michael Görög Several? 212 was updated on Nov 30th, 218 We can't do nothing if Hub doesn't provide a copy/paste option, 216 should have been fixed two weeks ago. Others? Meantime I will prepare another build for the Hub.
+
+``` + +Michael Görög +```html +
Corrado Cavalli: For example the bug where on the Hub was the singel user interface when starting the app the first time (212) or the issue with DropBox (218) where it is not really possible to use it on the Hub, because the code can't be copied.  Or (216) where we got a Panda when doing copy & paste from google drive. I am not sure if these are really Hub related but we spotted these bugs, when doing the test on the Hub.
+
+``` + +Corrado Cavalli +```html +
Michael Görög What do you mean with Hub related?
+
+``` + +Gian Paolo Santopaolo +```html +
Anything that can help is ok. Corrado Cavalli how is possible bugs only on the Hub?
+``` diff --git a/src/backend/temp/cd23dcdc7c3a44ee9e1a4fa2833b7732_.md b/src/backend/temp/cd23dcdc7c3a44ee9e1a4fa2833b7732_.md new file mode 100644 index 00000000..cd27bb97 --- /dev/null +++ b/src/backend/temp/cd23dcdc7c3a44ee9e1a4fa2833b7732_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/dates.xlsx b/src/backend/temp/dates.xlsx new file mode 100644 index 00000000..bd10d4b5 Binary files /dev/null and b/src/backend/temp/dates.xlsx differ diff --git a/src/backend/temp/e220589762a3426b84972288fd5251e8_.md b/src/backend/temp/e220589762a3426b84972288fd5251e8_.md new file mode 100644 index 00000000..8fa15f42 --- /dev/null +++ b/src/backend/temp/e220589762a3426b84972288fd5251e8_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/e80e6f17b4244df0ab50d256524618ac_.md b/src/backend/temp/e80e6f17b4244df0ab50d256524618ac_.md new file mode 100644 index 00000000..2f9eecf9 --- /dev/null +++ b/src/backend/temp/e80e6f17b4244df0ab50d256524618ac_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/e9b7d77ae33643b78818c6377a0f192d_.md b/src/backend/temp/e9b7d77ae33643b78818c6377a0f192d_.md new file mode 100644 index 00000000..8d747ca5 --- /dev/null +++ b/src/backend/temp/e9b7d77ae33643b78818c6377a0f192d_.md @@ -0,0 +1,22 @@ +Michael Görög +```html +
Dev Team: It seems there is a problem with CollaBoard. When starting the app we (and partners and customers) get this error messages:
+
+``` + +Michael Görög +```html +Now it is working again. +``` + +Michael Görög +```html +
+
+
 
+
+ +
 
+
+
+``` diff --git a/src/backend/temp/ea6639139f8945888fe8d23d75675e65_.md b/src/backend/temp/ea6639139f8945888fe8d23d75675e65_.md new file mode 100644 index 00000000..9852e5cf --- /dev/null +++ b/src/backend/temp/ea6639139f8945888fe8d23d75675e65_.md @@ -0,0 +1,30 @@ +Gian Paolo Santopaolo +```html +
Dev Team I've talked with Corrado Cavalli and by the end of this week we are trying to release both CB (without realtime but fully working stand alone user) and DE for beta to test. Andreas Görög it would be great if you can arrange some testing sessions for next week so that also Corrado can join (if needed) 
+
+``` + +Corrado Cavalli +```html +
+
+
Andreas Görög Gian Paolo Santopaolo Both CB (w/o realtime) and DE (full) are now pushed on relative dev/vnext branches. You can pull both (main app and IBV_Commons) and compile them.
+ +
Since it has a massive refactoring, after pulling, close and reopen Visual Studio and clean the solution before build the project.
+ +
At the moment DevOps build fails with multitargeting projects, I'm investigating on it.
+ +
To be clear: No new implementation, just moved to latest SDK 1809, so Windows 17773 is required to run (Hub is out) and ported, where possible, all code to .NET Standard for later reuse.
+ +
We need now to identify any regression introduced by this massive code refactoring.
+
+
+
+``` + +Andreas Görög +```html +
+
Let me know how we can pull the newest server and client versions. Actually we still have the version before the porting in the local sln's and have not pulled anything since then.
+
+``` diff --git a/src/backend/temp/ed77cce201c44fee8384a6f349821c7e_.md b/src/backend/temp/ed77cce201c44fee8384a6f349821c7e_.md new file mode 100644 index 00000000..f0f347c6 --- /dev/null +++ b/src/backend/temp/ed77cce201c44fee8384a6f349821c7e_.md @@ -0,0 +1,4 @@ + +```html + +``` diff --git a/src/backend/temp/f09bf6956d2249ccaa3193c43dea181b_.md b/src/backend/temp/f09bf6956d2249ccaa3193c43dea181b_.md new file mode 100644 index 00000000..0e0b3a2e --- /dev/null +++ b/src/backend/temp/f09bf6956d2249ccaa3193c43dea181b_.md @@ -0,0 +1,16 @@ +Michael Görög +```html +
Corrado Cavalli: Hi Corrado, is this the new client for Midor with the adjustments? +
+
 
+
+ +
If yes I would like to send it to Midor today. 
+
+
+``` + +Corrado Cavalli +```html +Yes it is, I also send it to the store. +``` diff --git a/src/backend/temp/fb2d4b3bb15340a9aa65d5afd42d27f4_.md b/src/backend/temp/fb2d4b3bb15340a9aa65d5afd42d27f4_.md new file mode 100644 index 00000000..f2a4a3f3 --- /dev/null +++ b/src/backend/temp/fb2d4b3bb15340a9aa65d5afd42d27f4_.md @@ -0,0 +1,78 @@ +Gian Paolo Santopaolo +```html +
+
+
+
+
Andreas Görög wanted to give you and Dev Team an update about development status:
+
+
+
+
+
+``` + +Gian Paolo Santopaolo +```html +
+
Corrado Cavalli Andreas Görög there is an update about this bug, it seems fixed and fix will be released after UniversalWindowsPlatform NuGet package version 6.2.3. see https://github.com/dotnet/corefx/issues/33704.
+ + +
So we have to wait  for this
+
+``` + +Gian Paolo Santopaolo +```html +
+
+ +
+ +
    + + +
  1. SignalR is actually not working and we are looking for a solution with the .Net Core Team, see conversation here
    + + https://github.com/SignalR/SignalR/issues/4272
    + + and here https://github.com/dotnet/corefx/issues/33704
    + +   +
  2. Cryptography is working but it needs to be tested deeply +
  3. CollaBoard is already working (except for the realtime part) +
  4. DocumentExplorer shoud be working today? Corrado Cavalli updates on timings? +
  5. Implementation is ok for Windows, several parts hast to be written (almost from scratch for Android and/or iOS). +
+ + +
Next Work to do (after DE working)
+ +
+ +
    + + +
  • Extract ViewModels and other code that now is I n the App's project and refactor to .Net Standard 2.0 library +
  • Start Xamarin development for other platforms +
  •   +
+ +
+
+
+
+
+``` + +Gian Paolo Santopaolo +```html +The most of the porting to .Net standard is done. There are some problems though:  +``` + +Gian Paolo Santopaolo +```html +
+
We have created a one note file where we are storing all the notes, you can find it here DevNotes  (Web view)
+
+``` diff --git a/src/backend/temp/fc2744e4c68a4fcba03510b525fed6a9_.md b/src/backend/temp/fc2744e4c68a4fcba03510b525fed6a9_.md new file mode 100644 index 00000000..5ed47406 --- /dev/null +++ b/src/backend/temp/fc2744e4c68a4fcba03510b525fed6a9_.md @@ -0,0 +1,70 @@ +Michael Görög +```html +
Corrado Cavalli: I downloaded this build and tested if I can login to the IBV environment. +
+
 
+
+ +
But I get the error message, that the username is not valid. 
+
+
+``` + +Michael Görög +```html +
Corrado Cavalli: Thank you. Now I can login, but not connect to the server. I will check with Oli if Server and Storage address are correct.
+
+``` + +Corrado Cavalli +```html +
Michael Görög Ah... so that is not the right build... you should have received one yesterday evening with IBV ON PREMISE
+
+``` + +Michael Görög +```html +
Corrado Cavalli: Yes this is working, but we need to test it internaly to check if DFS is working or not. So we need a version where we are able to connect to Z. 
+
+``` + +Corrado Cavalli +```html +
Michael Görög does it work with Guest1/... user?
+
+``` + +Michael Görög +```html +
Corrado Cavalli: I installed 1.3.61. If I click on AD I still see the Guest user. When I try to connect using my AD credentials, I still get the same error: +
+
 
+
+ +
 
+
+
+``` + +Corrado Cavalli +```html +
Michael Görög Let me schedule a new build (61) and let me know if it works with that...
+
+``` + +Michael Görög +```html +
Corrado Cavalli: Yes I did. I tested on another device as well (uninstall and reinstall) but I have the issue that I cant login. What other options do we have? (Just for your Info, when I click on AD Login, User Name, Passwort and Domain are already filled with the Guest1 User).
+
+``` + +Corrado Cavalli +```html +
Michael Görög build is right, that commit refer to IBV on premise server, did you try uninstall previous version?
+
+``` + +Michael Görög +```html +Can you please check if this is the right build or not? +``` diff --git a/src/backend/temp/project_templates_how_to_import_BackOffice.pdf b/src/backend/temp/project_templates_how_to_import_BackOffice.pdf new file mode 100644 index 00000000..638894c6 Binary files /dev/null and b/src/backend/temp/project_templates_how_to_import_BackOffice.pdf differ diff --git a/src/backend/temp/small.pdf b/src/backend/temp/small.pdf new file mode 100644 index 00000000..c1c12fbd Binary files /dev/null and b/src/backend/temp/small.pdf differ diff --git a/src/backend/temp/template_projects_per_category.xlsx b/src/backend/temp/template_projects_per_category.xlsx new file mode 100644 index 00000000..2c502ad0 Binary files /dev/null and b/src/backend/temp/template_projects_per_category.xlsx differ diff --git a/src/backend/whisper/lib/gen_types/whisper_data_pb2.py b/src/backend/whisper/lib/gen_types/whisper_data_pb2.py new file mode 100644 index 00000000..7363677b --- /dev/null +++ b/src/backend/whisper/lib/gen_types/whisper_data_pb2.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: whisper_data.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import file_type_pb2 as file__type__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12whisper_data.proto\x12\ncom.cognix\x1a\x0f\x66ile_type.proto\"\xb4\x01\n\x0bWhisperData\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x03\x12\x14\n\x0c\x63onnector_id\x18\x03 \x01(\x03\x12\'\n\tfile_type\x18\x04 \x01(\x0e\x32\x14.com.cognix.FileType\x12\x17\n\x0f\x63ollection_name\x18\x05 \x01(\t\x12\x12\n\nmodel_name\x18\x06 \x01(\t\x12\x17\n\x0fmodel_dimension\x18\x07 \x01(\x05\x42\x1aZ\x18\x62\x61\x63kend/core/proto;protob\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'whisper_data_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z\030backend/core/proto;proto' + _globals['_WHISPERDATA']._serialized_start=52 + _globals['_WHISPERDATA']._serialized_end=232 +# @@protoc_insertion_point(module_scope) diff --git a/src/backend/whisper/lib/gen_types/whisper_data_pb2.pyi b/src/backend/whisper/lib/gen_types/whisper_data_pb2.pyi new file mode 100644 index 00000000..812771a4 --- /dev/null +++ b/src/backend/whisper/lib/gen_types/whisper_data_pb2.pyi @@ -0,0 +1,24 @@ +import file_type_pb2 as _file_type_pb2 +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class WhisperData(_message.Message): + __slots__ = ("url", "document_id", "connector_id", "file_type", "collection_name", "model_name", "model_dimension") + URL_FIELD_NUMBER: _ClassVar[int] + DOCUMENT_ID_FIELD_NUMBER: _ClassVar[int] + CONNECTOR_ID_FIELD_NUMBER: _ClassVar[int] + FILE_TYPE_FIELD_NUMBER: _ClassVar[int] + COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int] + MODEL_NAME_FIELD_NUMBER: _ClassVar[int] + MODEL_DIMENSION_FIELD_NUMBER: _ClassVar[int] + url: str + document_id: int + connector_id: int + file_type: _file_type_pb2.FileType + collection_name: str + model_name: str + model_dimension: int + def __init__(self, url: _Optional[str] = ..., document_id: _Optional[int] = ..., connector_id: _Optional[int] = ..., file_type: _Optional[_Union[_file_type_pb2.FileType, str]] = ..., collection_name: _Optional[str] = ..., model_name: _Optional[str] = ..., model_dimension: _Optional[int] = ...) -> None: ... diff --git a/src/backend/whisper/lib/gen_types/whisper_data_pb2_grpc.py b/src/backend/whisper/lib/gen_types/whisper_data_pb2_grpc.py new file mode 100644 index 00000000..73c0894b --- /dev/null +++ b/src/backend/whisper/lib/gen_types/whisper_data_pb2_grpc.py @@ -0,0 +1,29 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in whisper_data_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) diff --git a/src/backend/whisper/whisper_service.py b/src/backend/whisper/whisper_service.py new file mode 100644 index 00000000..0277f7d8 --- /dev/null +++ b/src/backend/whisper/whisper_service.py @@ -0,0 +1,183 @@ +#region imports +import asyncio +import logging +import os +import threading +import time +import datetime +from dotenv import load_dotenv +from nats.aio.msg import Msg + +from lib.db.db_connector import ConnectorCRUD, Status +from lib.db.db_document import DocumentCRUD +from lib.gen_types.semantic_data_pb2 import SemanticData +from lib.semantic.semantic_factory import SemanticFactory +from lib.db.jetstream_event_subscriber import JetStreamEventSubscriber +from readiness_probe import ReadinessProbe + +#endregion + +#region .env and logs +# Load environment variables from .env file +load_dotenv() + +# get log level from env +log_level_str = os.getenv('LOG_LEVEL', 'ERROR').upper() +log_level = getattr(logging, log_level_str, logging.INFO) +# get log format from env +log_format = os.getenv('LOG_FORMAT', '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s') +# Configure logging +logging.basicConfig(level=log_level, format=log_format) + +logger = logging.getLogger(__name__) +logger.info(f"Logging configured with level {log_level_str} and format {log_format}") + +# loading from env env +nats_url = os.getenv('NATS_CLIENT_URL', 'nats://127.0.0.1:4222') +nats_connect_timeout = int(os.getenv('NATS_CLIENT_CONNECT_TIMEOUT', '30')) +nats_reconnect_time_wait = int(os.getenv('NATS_CLIENT_RECONNECT_TIME_WAIT', '30')) +nats_max_reconnect_attempts = int(os.getenv('NATS_CLIENT_MAX_RECONNECT_ATTEMPTS', '3')) +semantic_stream_name = os.getenv('NATS_CLIENT_WHISPER_STREAM_NAME', 'whisper') +semantic_stream_subject = os.getenv('NATS_CLIENT_WHISPER_STREAM_SUBJECT', 'whisper_activity') +semantic_ack_wait = int(os.getenv('NATS_CLIENT_WHISPER_ACK_WAIT', '3600')) # seconds +semantic_max_deliver = int(os.getenv('NATS_CLIENT_SEMANTIC_MAX_DELIVER', '3')) + +cockroach_url = os.getenv('COCKROACH_CLIENT_DATABASE_URL', + 'postgres://root:123@cockroach:26257/defaultdb?sslmode=disable') + + +#endregion + +# Define the event handler function +async def whisper_event(msg: Msg): + start_time = time.time() # Record the start time + connector_id = 0 + entities_analyzed = 0 + try: + logger.info("🔥 starting semantic analysis.. new version..") + # Deserialize the message + semantic_data = SemanticData() + semantic_data.ParseFromString(msg.data) + logger.info(f"message: \n {semantic_data}") + if semantic_data.model_name == "": + logger.error(f"❌ no model nameeeeeeee") + semantic_data.model_name = "paraphrase-multilingual-mpnet-base-v2" + semantic_data.model_dimension = 768 + logger.warning(f"😱 Addning model name and dimension manually remove this code ASAP") + + # verify document id is valid otherwise we cannot process the message + if semantic_data.document_id <= 0: + logger.error(f"❌ failed to process semantic data error: document_id must value must be positive") + else: + # see if doc exists + document_crud = DocumentCRUD(cockroach_url) + document = document_crud.select_document(semantic_data.document_id) + + if document: + # needed for th finally block + connector_id = document.connector_id + + # update connector's status + connector_crud = ConnectorCRUD(cockroach_url) + connector = connector_crud.select_connector(document.connector_id) + last_successful_index_date = connector.last_successful_analyzed + connector_crud.update_connector(document.connector_id, + status=Status.PROCESSING, + last_update=datetime.datetime.utcnow()) + + # performing semantic analysis on the source + semantic = SemanticFactory.create_semantic_analyzer(semantic_data.file_type) + entities_analyzed = await semantic.analyze(data=semantic_data, full_process_start_time=start_time, + ack_wait=semantic_ack_wait, cockroach_url=cockroach_url) + + # if entities_analyzed == 0 this means no data was stored in the vector db + # we shall find a way to tell the user, most likely put the message in the dead letter + + if entities_analyzed is None: + logger.error(f"❌ entities_analyzed is none!!!!!") + entities_analyzed = 0 + # updating again the connector + a = connector_crud.update_connector(connector_id, + status=Status.COMPLETED_SUCCESSFULLY, + last_successful_analyzed=datetime.datetime.utcnow(), + last_update=datetime.datetime.utcnow(), + total_docs_analyzed=entities_analyzed + # TODO: we are storing the total entities in total docs. one doc will probably generate more than one chunk + ) + logger.info(a) + else: + logger.error( + f"❌ failed to process semantic data error: document_id {semantic_data.document_id} not valid") + # Acknowledge the message when done + await msg.ack_sync() + logger.info(f"👍 message acknowledged successfully, total entities stored {entities_analyzed}") + except Exception as e: + error_message = str(e) if e else "Unknown error occurred" + logger.error(f"❌ failed to process semantic data error: {error_message}") + if msg: # Ensure msg is not None before awaiting + await msg.nak() + try: + if connector_id != 0: + connector_crud = ConnectorCRUD(cockroach_url) + connector_crud.update_connector(connector_id, + status=Status.COMPLETED_WITH_ERRORS, + last_update=datetime.datetime.utcnow()) + except Exception as e: + error_message = str(e) if e else "Unknown error occurred" + logger.error(f"❌ failed to process semantic data error: {error_message}") + finally: + end_time = time.time() # Record the end time + elapsed_time = end_time - start_time + logger.info(f"⏰⏰ total elapsed time: {elapsed_time:.2f} seconds") + + +# TODO: IMPORTANT WHEN IT DOES NOT CONNECT TO COCKROACH IS PROCESSING!!!!! +# Andri shall make a fix query on the db if status is processing and max ack wait is more than last update +# then it means the process hanged but if max retries (from nats) has not reached it's limit +# then nats will post again the message +# orchestrator shall update to UNABLE_TO_PROCESS, if nats will post again the message this service will +# care to set the correct status + +async def main(): + # Start the readiness probe server in a separate thread + readiness_probe = ReadinessProbe() + readiness_probe_thread = threading.Thread(target=readiness_probe.start_server, daemon=True) + readiness_probe_thread.start() + + # circuit breaker for chunking + # if for reason nats won't be available + # semantic will wait till nats will be up again + while True: + logger.info("🛠️ service starting..") + try: + # subscribing to jet stream + subscriber = JetStreamEventSubscriber( + nats_url=nats_url, + stream_name=semantic_stream_name, + subject=semantic_stream_subject, + connect_timeout=nats_connect_timeout, + reconnect_time_wait=nats_reconnect_time_wait, + max_reconnect_attempts=nats_max_reconnect_attempts, + ack_wait=semantic_ack_wait, + max_deliver=semantic_max_deliver, + proto_message_type=SemanticData + ) + + subscriber.set_event_handler(whisper_event) + await subscriber.connect_and_subscribe() + + # todo add an event to JetStreamEventSubscriber to signal that connection has been established + logger.info("🚀 service started successfully") + + while True: + await asyncio.sleep(1) + + except KeyboardInterrupt: + logger.info("🛑 Service is stopping due to keyboard interrupt") + except Exception as e: + logger.exception(f"💀 recovering from a fatal error: {e}. The process will restart in 5 seconds..") + await asyncio.sleep(5) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/bot/readme.md b/src/bot/readme.md new file mode 100644 index 00000000..ee685f58 --- /dev/null +++ b/src/bot/readme.md @@ -0,0 +1,11 @@ +This will be the UI of CogniX-Bot + +We will expose OpenAI compatible APIs +In the UI the customer can see the chat history by session +can configure LLM and all the toher feature of cognix +There will not be any Chat UI + +We then need to see how many chat plug ins aready exists +Eventually contacting them, if needed, to see if they want to support CogniX by design + +might also be a different github project \ No newline at end of file diff --git a/src/config/.env.example b/src/config/.env.example new file mode 100644 index 00000000..041072c1 --- /dev/null +++ b/src/config/.env.example @@ -0,0 +1,4 @@ +GOOGLE_CLIENT_ID= +GOOGLE_SECRET= +MICROSOFT_CLIENT_ID= +MICROSOFT_CLIENT_SECRET= diff --git a/src/config/api_srv.env b/src/config/api_srv.env new file mode 100644 index 00000000..d17b7bcc --- /dev/null +++ b/src/config/api_srv.env @@ -0,0 +1,5 @@ +PORT=8080 +GOOGLE_CLIENT_ID="" +GOOGLE_SECRET="" +MICROSOFT_CLIENT_ID="" +MICROSOFT_CLIENT_SECRET="" \ No newline at end of file diff --git a/src/config/cockroach_cli.env b/src/config/cockroach_cli.env new file mode 100644 index 00000000..3bf71b42 --- /dev/null +++ b/src/config/cockroach_cli.env @@ -0,0 +1,4 @@ +COCKROACH_CLIENT_DATABASE_URL_GOLANG=postgres://root:123@cockroach:26257/defaultdb?sslmode=disable +COCKROACH_CLIENT_DATABASE_URL='cockroachdb://root:123@cockroach:26257/defaultdb?sslmode=disable' +COCKROACH_CLIENT_DB_DEBUG=false + diff --git a/src/config/embedder_cli.env b/src/config/embedder_cli.env new file mode 100644 index 00000000..47eda7e5 --- /dev/null +++ b/src/config/embedder_cli.env @@ -0,0 +1,7 @@ +EMBEDDER_GRPC_HOST=embedder +EMBEDDER_GRPC_PORT=50051 + + +# values define default embedding model +DEFAULT_EMBEDDING_MODEL=paraphrase-multilingual-mpnet-base-v2 +DEFAULT_EMBEDDING_VECTOR_SIZE=768 diff --git a/src/config/embedder_srv.env b/src/config/embedder_srv.env new file mode 100644 index 00000000..d63787fc --- /dev/null +++ b/src/config/embedder_srv.env @@ -0,0 +1,6 @@ +LOG_LEVEL=INFO +LOG_FORMAT='%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s' +# max number of sentence transformer instances, could be 1gb+ each +MODEL_CACHE_LIMIT=2 +LOCAL_MODEL_PATH=../models +GRPC_PORT=50051 diff --git a/src/config/grafana-ds.yaml b/src/config/grafana-ds.yaml new file mode 100644 index 00000000..8b16d206 --- /dev/null +++ b/src/config/grafana-ds.yaml @@ -0,0 +1,21 @@ +apiVersion: 1 +datasources: +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: false + +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false \ No newline at end of file diff --git a/src/config/kk.json b/src/config/kk.json new file mode 100644 index 00000000..b17cc763 --- /dev/null +++ b/src/config/kk.json @@ -0,0 +1,23872 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 1, + "id": 1, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "GitHub", + "type": "link", + "url": "https://github.com/rfmoz/grafana-dashboards" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Grafana", + "type": "link", + "url": "https://grafana.com/grafana/dashboards/1860" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Quick CPU / Mem / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Resource pressure via PSI", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 323, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "CPU", + "range": false, + "refId": "CPU some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "Mem", + "range": false, + "refId": "Memory some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "I/O", + "range": false, + "refId": "I/O some", + "step": 240 + } + ], + "title": "Pressure", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 20, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", instance=\"$node\"}[$__rate_interval])))", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "System load over all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 155, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "scalar(node_load1{instance=\"$node\",job=\"$job\"}) * 100 / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "hideTimeOverride": false, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\", job=\"$job\"}) / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"}) * 100", + "format": "time_series", + "hide": true, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(1 - (node_memory_MemAvailable_bytes{instance=\"$node\", job=\"$job\"} / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"})) * 100", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "B", + "step": 240 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 21, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"})) * 100", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 154, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 14, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total RootFS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 23, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 75, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 263, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SWAP Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEDAF7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Used", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Cache + Buffer", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Free", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SWAP Used", + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk space used of all filesystems mounted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used Basic", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 265, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Iowait - Waiting for I/O to complete" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Irq - Servicing interrupts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nice - Niced processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Softirq - Servicing softirqs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System - Processes executing in kernel mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User - Normal processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User - Normal processes executing in user mode", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "refId": "J", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap - Swap memory usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 24, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "title": "Memory Stack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bits out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 84, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 156, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 229, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*read*./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 42, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "I/O Usage Read / Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%util", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 127, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "I/O Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Guest - /" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195ce", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^GuestNice - /" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#c15c17", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", + "hide": false, + "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", + "hide": false, + "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", + "range": true, + "refId": "B" + } + ], + "title": "CPU spent seconds in guests (VMs)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 266, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 136, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Active / Inactive", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 135, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Committed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 191, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Writeback and Dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 138, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 131, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 70, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 159, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "refId": "A", + "step": 240 + } + ], + "title": "Memory Bounce", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive *./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Anonymous", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 160, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Kernel / CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 140, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "C", + "step": 240 + } + ], + "title": "Memory HugePages Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 71, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "B", + "step": 240 + } + ], + "title": "Memory HugePages Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 128, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 240 + } + ], + "title": "Memory DirectMap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 137, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Unevictable and MLocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 124 + }, + "id": 132, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage", + "refId": "A", + "step": 240 + } + ], + "title": "Memory NFS", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 267, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 176, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages Swap In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "faults", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault operations" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 175, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Page Faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 307, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "oom killer invocations ", + "refId": "A", + "step": 240 + } + ], + "title": "OOM Killer", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 293, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 260, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Estimated error in seconds", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "title": "Time Synchronized Drift", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 291, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "title": "Time PLL Adjust", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 168, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "title": "Time Synchronized Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 294, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "title": "Time Misc", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Timesync", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 312, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 62, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Enable with --collector.processes argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 315, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "refId": "A", + "step": 240 + } + ], + "title": "Processes State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "forks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 148, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes forks second", + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 149, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "D", + "step": 240 + } + ], + "title": "Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Enable with --collector.processes argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PIDs limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 313, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Number of PIDs", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PIDs limit", + "refId": "B", + "step": 240 + } + ], + "title": "PIDs Number and Limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 93 + }, + "id": 305, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "refId": "B", + "step": 240 + } + ], + "title": "Process schedule stats Running / Waiting", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Enable with --collector.processes argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Threads limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 314, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated threads", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Threads limit", + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 269, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 240 + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 321, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Min", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Frequency Scaling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "https://docs.kernel.org/accounting/psi.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Memory some" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory full" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I/O some" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I/O full" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 322, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CPU some", + "range": true, + "refId": "CPU some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory some", + "range": true, + "refId": "Memory some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory full", + "range": true, + "refId": "Memory full", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O some", + "range": true, + "refId": "I/O some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O full", + "range": true, + "refId": "I/O full", + "step": 240 + } + ], + "title": "Pressure Stall Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Enable with --collector.interrupts argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 259, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "title": "Interrupts Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 306, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "refId": "A", + "step": 240 + } + ], + "title": "Schedule timeslices executed by each cpu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 151, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 308, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time spent", + "refId": "A", + "step": 240 + } + ], + "title": "CPU time spent in user and system contexts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 64, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptors", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 304, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "temperature", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 158, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "title": "Hardware temperature monitor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 300, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Current {{ name }} in {{ type }}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max {{ name }} in {{ type }}", + "refId": "B", + "step": 240 + } + ], + "title": "Throttle cooling device", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 302, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ power_supply }} online", + "refId": "A", + "step": 240 + } + ], + "title": "Power supply", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 296, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 297, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }} Connections", + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFCB7D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 298, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Activating", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Deactivating", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inactive", + "refId": "E", + "step": 240 + } + ], + "title": "Systemd Units State", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Systemd", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 270, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number (after merges) of I/O requests completed per second for the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Completed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 33, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time. read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 37, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read wait time avg", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write wait time avg", + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "aqu-sz", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 35, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "I/Os", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 133, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read merged", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%util", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 36, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - discard", + "refId": "B", + "step": 240 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Outstanding req.", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 240 + } + ], + "title": "Instantaneous Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 301, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Discards completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Discards completed / merged", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Storage Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 271, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 43, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Size", + "refId": "C", + "step": 240 + } + ], + "title": "Filesystem space available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 41, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free file nodes", + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "files", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 28, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open files", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 219, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - File nodes total", + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "/ ReadOnly" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 44, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Device error", + "refId": "B", + "step": 240 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 272, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 142, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive errors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit errors", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 143, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive drop", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit drop", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 141, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive compressed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 146, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive multicast", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 144, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive fifo", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 145, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive frame", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 231, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Carrier", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 232, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit colls", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Colls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack entries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack limit", + "refId": "B", + "step": 240 + } + ], + "title": "NF Conntrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 230, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - ARP entries", + "refId": "A", + "step": 240 + } + ], + "title": "ARP Entries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 97 + }, + "id": 288, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Bytes", + "refId": "A", + "step": 240 + } + ], + "title": "MTU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 280, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Speed", + "refId": "A", + "step": 240 + } + ], + "title": "Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 289, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Interface transmit queue length", + "refId": "A", + "step": 240 + } + ], + "title": "Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packetes drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 117 + }, + "id": 290, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Processed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Dropped", + "refId": "B", + "step": 240 + } + ], + "title": "Softnet Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 117 + }, + "id": 310, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 240 + } + ], + "title": "Softnet Out of Quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 127 + }, + "id": 309, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{interface}} - Operational state UP", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link state", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 273, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 63, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_alloc - Allocated sockets", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_tw - Sockets waiting close", + "refId": "E", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 124, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 125, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 220, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "sockets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 126, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 274, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "octets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 221, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InOctets - Received octets", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "OutOctets - Sent octets", + "refId": "B", + "step": 240 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 81, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Forwarding - IP forwarding", + "refId": "A", + "step": 240 + } + ], + "title": "Netstat IP Forwarding", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 115, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 240 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 50, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InDatagrams - Datagrams received", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutDatagrams - Datagrams sent", + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 109, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 299, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "B", + "step": 240 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 104, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "OutRsts - Segments sent with RST flag", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPRcvQDrop{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "TCPRcvQDrop - Packets meant to be queued in rcv queue but dropped because socket rcvbuf limit hit", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "TCPOFOQueue - TCP layer receives an out of order packet and has enough memory to queue it", + "range": true, + "refId": "H" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*MaxConn *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 85, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")", + "refId": "B", + "step": 240 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Sent.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 91, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 82, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "B", + "step": 240 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Enable with --collector.tcpstat argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 320, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "established - TCP sockets in established state", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "fin_wait2 - TCP sockets in fin_wait2 state", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "listen - TCP sockets in listen state", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "time_wait - TCP sockets in time_wait state", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "TCP Stat", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 279, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 40, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape duration", + "refId": "A", + "step": 240 + } + ], + "title": "Node Exporter Scrape Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*error.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 157, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape success", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 240 + } + ], + "title": "Node Exporter Scrape", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Node Exporter", + "type": "row" + } + ], + "refresh": "1m", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "ado2yg93bkutcf" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "node-exporter", + "value": "node-exporter" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(node_uname_info, job)", + "refId": "Prometheus-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "node-exporter:9100", + "value": "node-exporter:9100" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\"}, instance)", + "refId": "Prometheus-node-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "rYdddlPWk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/src/config/loki-config.yaml b/src/config/loki-config.yaml new file mode 100644 index 00000000..e448dfd9 --- /dev/null +++ b/src/config/loki-config.yaml @@ -0,0 +1,50 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false diff --git a/src/config/milvus_cli.env b/src/config/milvus_cli.env new file mode 100644 index 00000000..f6376bc1 --- /dev/null +++ b/src/config/milvus_cli.env @@ -0,0 +1,15 @@ +MILVUS_URL=milvus:19530 +MILVUS_METRIC_TYPE=COSINE +MILVUS_INDEX_STRATEGY=DISKANN +# python +MILVUS_ALIAS=default +# when running inside the cluster: use the name of the milvus instance inside the cluster +# when running locally: use 127.0.0.1 +MILVUS_HOST=milvus-standalone # the name of the milvus instance inside the cluster +MILVUS_PORT=19530 + +MILVUS_INDEX_TYPE=DISKANN +MILVUS_METRIC_TYPE=COSINE + +MILVUS_USERNAME=root +MILVUS_PASSWORD=sq5/6<$Y4aD`2;Gba'E# diff --git a/src/config/milvus_srv.yaml b/src/config/milvus_srv.yaml new file mode 100644 index 00000000..a8ce228a --- /dev/null +++ b/src/config/milvus_srv.yaml @@ -0,0 +1,677 @@ +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Related configuration of etcd, used to store Milvus metadata & service discovery. +etcd: + endpoints: localhost:2379 + rootPath: by-dev # The root path where data is stored in etcd + metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath + kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath + log: + level: warn # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + # path is one of: + # - "default" as os.Stderr, + # - "stderr" as os.Stderr, + # - "stdout" as os.Stdout, + # - file path to append server logs to. + # please adjust in embedded Milvus: /tmp/milvus/logs/etcd.log + path: stdout + # ssl: + # enabled: false # Whether to support ETCD secure connection mode + # tlsCert: /path/to/etcd-client.pem # path to your cert file + # tlsKey: /path/to/etcd-client-key.pem # path to your key file + # tlsCACert: /path/to/ca.pem # path to your CACert file + # # TLS min version + # # Optional values: 1.0, 1.1, 1.2, 1.3。 + # # We recommend using version 1.2 and above. + # tlsMinVersion: 1.3 + requestTimeout: 10000 # Etcd operation timeout in milliseconds + use: + embed: false # Whether to enable embedded Etcd (an in-process EtcdServer). + data: + dir: default.etcd # Embedded Etcd only. please adjust in embedded Milvus: /tmp/milvus/etcdData/ + auth: + enabled: false # Whether to enable authentication + userName: # username for etcd authentication + password: # password for etcd authentication + +metastore: + type: etcd # Default value: etcd, Valid values: [etcd, tikv] + +localStorage: + path: /var/lib/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/ + +# Related configuration of MinIO/S3/GCS or any other service supports S3 API, which is responsible for data persistence for Milvus. +# We refer to the storage service as MinIO/S3 in the following description for simplicity. +minio: + address: localhost # Address of MinIO/S3 + port: 9000 # Port of MinIO/S3 + accessKeyID: minioadmin # accessKeyID of MinIO/S3 + secretAccessKey: minioadmin # MinIO/S3 encryption string + useSSL: false # Access to MinIO/S3 with SSL + ssl: + tlsCACert: /path/to/public.crt # path to your CACert file + bucketName: a-bucket # Bucket name in MinIO/S3 + rootPath: files # The root path where the message is stored in MinIO/S3 + # Whether to useIAM role to access S3/GCS instead of access/secret keys + # For more information, refer to + # aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html + # gcp: https://cloud.google.com/storage/docs/access-control/iam + # aliyun (ack): https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/use-rrsa-to-enforce-access-control + # aliyun (ecs): https://www.alibabacloud.com/help/en/elastic-compute-service/latest/attach-an-instance-ram-role + useIAM: false + # Cloud Provider of S3. Supports: "aws", "gcp", "aliyun". + # You can use "aws" for other cloud provider supports S3 API with signature v4, e.g.: minio + # You can use "gcp" for other cloud provider supports S3 API with signature v2 + # You can use "aliyun" for other cloud provider uses virtual host style bucket + # When useIAM enabled, only "aws", "gcp", "aliyun" is supported for now + cloudProvider: aws + # Custom endpoint for fetch IAM role credentials. when useIAM is true & cloudProvider is "aws". + # Leave it empty if you want to use AWS default endpoint + iamEndpoint: + logLevel: fatal # Log level for aws sdk log. Supported level: off, fatal, error, warn, info, debug, trace + region: # Specify minio storage system location region + useVirtualHost: false # Whether use virtual host mode for bucket + requestTimeoutMs: 10000 # minio timeout for request time in milliseconds + # The maximum number of objects requested per batch in minio ListObjects rpc, + # 0 means using oss client by default, decrease these configration if ListObjects timeout + listObjectsMaxKeys: 0 + + +# natsmq configuration. +# more detail: https://docs.nats.io/running-a-nats-service/configuration +natsmq: + server: + port: 4222 # Port for nats server listening + storeDir: /var/lib/milvus/nats # Directory to use for JetStream storage of nats + maxFileStore: 17179869184 # Maximum size of the 'file' storage + maxPayload: 8388608 # Maximum number of bytes in a message payload + maxPending: 67108864 # Maximum number of bytes buffered for a connection Applies to client connections + initializeTimeout: 4000 # waiting for initialization of natsmq finished + monitor: + trace: false # If true enable protocol trace log messages + debug: false # If true enable debug log messages + logTime: true # If set to false, log without timestamps. + logFile: /tmp/milvus/logs/nats.log # Log file path relative to .. of milvus binary if use relative path + logSizeLimit: 536870912 # Size in bytes after the log file rolls over to a new one + retention: + maxAge: 4320 # Maximum age of any message in the P-channel + maxBytes: # How many bytes the single P-channel may contain. Removing oldest messages if the P-channel exceeds this size + maxMsgs: # How many message the single P-channel may contain. Removing oldest messages if the P-channel exceeds this limit + + port: 19530 + internalPort: 19529 + grpc: + serverMaxSendSize: 268435456 + serverMaxRecvSize: 67108864 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 67108864 + +# Related configuration of queryCoord, used to manage topology and load balancing for the query nodes, and handoff from growing segments to sealed segments. +queryCoord: + taskMergeCap: 1 + taskExecutionCap: 256 + autoHandoff: true # Enable auto handoff + autoBalance: true # Enable auto balance + autoBalanceChannel: true # Enable auto balance channel + balancer: ScoreBasedBalancer # auto balancer used for segments on queryNodes + globalRowCountFactor: 0.1 # the weight used when balancing segments among queryNodes + scoreUnbalanceTolerationFactor: 0.05 # the least value for unbalanced extent between from and to nodes when doing balance + reverseUnBalanceTolerationFactor: 1.3 # the largest value for unbalanced extent between from and to nodes after doing balance + overloadedMemoryThresholdPercentage: 90 # The threshold percentage that memory overload + balanceIntervalSeconds: 60 + memoryUsageMaxDifferencePercentage: 30 + rowCountFactor: 0.4 # the row count weight used when balancing segments among queryNodes + segmentCountFactor: 0.4 # the segment count weight used when balancing segments among queryNodes + globalSegmentCountFactor: 0.1 # the segment count weight used when balancing segments among queryNodes + segmentCountMaxSteps: 50 # segment count based plan generator max steps + rowCountMaxSteps: 50 # segment count based plan generator max steps + randomMaxSteps: 10 # segment count based plan generator max steps + growingRowCountWeight: 4 # the memory weight of growing segment row count + balanceCostThreshold: 0.001 # the threshold of balance cost, if the difference of cluster's cost after executing the balance plan is less than this value, the plan will not be executed + checkSegmentInterval: 1000 + checkChannelInterval: 1000 + checkBalanceInterval: 10000 + checkIndexInterval: 10000 + channelTaskTimeout: 60000 # 1 minute + segmentTaskTimeout: 120000 # 2 minute + distPullInterval: 500 + heartbeatAvailableInterval: 10000 # 10s, Only QueryNodes which fetched heartbeats within the duration are available + loadTimeoutSeconds: 600 + distRequestTimeout: 5000 # the request timeout for querycoord fetching data distribution from querynodes, in milliseconds + heatbeatWarningLag: 5000 # the lag value for querycoord report warning when last heatbeat is too old, in milliseconds + checkHandoffInterval: 5000 + enableActiveStandby: false + checkInterval: 1000 + checkHealthInterval: 3000 # 3s, the interval when query coord try to check health of query node + checkHealthRPCTimeout: 2000 # 100ms, the timeout of check health rpc to query node + brokerTimeout: 5000 # 5000ms, querycoord broker rpc timeout + collectionRecoverTimes: 3 # if collection recover times reach the limit during loading state, release it + observerTaskParallel: 16 # the parallel observer dispatcher task number + checkAutoBalanceConfigInterval: 10 # the interval of check auto balance config + checkNodeSessionInterval: 60 # the interval(in seconds) of check querynode cluster session + gracefulStopTimeout: 5 # seconds. force stop node without graceful stop + enableStoppingBalance: true # whether enable stopping balance + channelExclusiveNodeFactor: 4 # the least node number for enable channel's exclusive mode + cleanExcludeSegmentInterval: 60 # the time duration of clean pipeline exclude segment which used for filter invalid data, in seconds + ip: # if not specified, use the first unicastable address + port: 19531 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +# Related configuration of queryNode, used to run hybrid search between vector and scalar data. +queryNode: + stats: + publishInterval: 1000 # Interval for querynode to report node information (milliseconds) + segcore: + knowhereThreadPoolNumRatio: 4 # The number of threads in knowhere's thread pool. If disk is enabled, the pool size will multiply with knowhereThreadPoolNumRatio([1, 32]). + # used for zilliz-cloud ; please ignore it for open source. + knowhereScoreConsistency: false + chunkRows: 128 # The number of vectors in a chunk. + interimIndex: + enableIndex: true # Enable segment build with index to accelerate vector search when segment is in growing or binlog. + nlist: 128 # temp index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 + nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist + memExpansionRate: 1.15 # extra memory needed by building interim index + buildParallelRate: 0.5 # the ratio of building interim index parallel matched with cpu num + loadMemoryUsageFactor: 1 # The multiply factor of calculating the memory usage while loading segments + enableDisk: false # enable querynode load disk index, and search on disk index + maxDiskUsagePercentage: 95 + cache: + enabled: true + memoryLimit: 2147483648 # 2 GB, 2 * 1024 *1024 *1024 + readAheadPolicy: willneed # The read ahead policy of chunk cache, options: `normal, random, sequential, willneed, dontneed` + # options: async, sync, disable. + # Specifies the necessity for warming up the chunk cache. + # 1. If set to "sync" or "async" the original vector data will be synchronously/asynchronously loaded into the + # chunk cache during the load process. This approach has the potential to substantially reduce query/search latency + # for a specific duration post-load, albeit accompanied by a concurrent increase in disk usage; + # 2. If set to "disable" original vector data will only be loaded into the chunk cache during search/query. + warmup: disable + mmap: + mmapEnabled: false # Enable mmap for loading data + lazyload: + enabled: false # Enable lazyload for loading data + waitTimeout: 30000 # max wait timeout duration in milliseconds before start to do lazyload search and retrieve + requestResourceTimeout: 5000 # max timeout in milliseconds for waiting request resource for lazy load, 5s by default + requestResourceRetryInterval: 2000 # retry interval in milliseconds for waiting request resource for lazy load, 2s by default + maxRetryTimes: 1 # max retry times for lazy load, 1 by default + maxEvictPerRetry: 1 # max evict count for lazy load, 1 by default + grouping: + enabled: true + maxNQ: 1000 + topKMergeRatio: 20 + scheduler: + receiveChanSize: 10240 + unsolvedQueueSize: 10240 + # maxReadConcurrentRatio is the concurrency ratio of read task (search task and query task). + # Max read concurrency would be the value of hardware.GetCPUNum * maxReadConcurrentRatio. + # It defaults to 2.0, which means max read concurrency would be the value of hardware.GetCPUNum * 2. + # Max read concurrency must greater than or equal to 1, and less than or equal to hardware.GetCPUNum * 100. + # (0, 100] + maxReadConcurrentRatio: 1 + cpuRatio: 10 # ratio used to estimate read task cpu usage. + maxTimestampLag: 86400 + scheduleReadPolicy: + # fifo: A FIFO queue support the schedule. + # user-task-polling: + # The user's tasks will be polled one by one and scheduled. + # Scheduling is fair on task granularity. + # The policy is based on the username for authentication. + # And an empty username is considered the same user. + # When there are no multi-users, the policy decay into FIFO" + name: fifo + taskQueueExpire: 60 # Control how long (many seconds) that queue retains since queue is empty + enableCrossUserGrouping: false # Enable Cross user grouping when using user-task-polling policy. (Disable it if user's task can not merge each other) + maxPendingTaskPerUser: 1024 # Max pending task per user in scheduler + dataSync: + flowGraph: + maxQueueLength: 16 # Maximum length of task queue in flowgraph + maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph + enableSegmentPrune: false # use partition prune function on shard delegator + queryStreamBatchSize: 4194304 # return batch size of stream query + ip: # if not specified, use the first unicastable address + port: 21123 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +indexCoord: + bindIndexNodeMode: + enable: false + address: localhost:22930 + withCred: false + nodeID: 0 + segment: + minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed + +indexNode: + scheduler: + buildParallel: 1 + enableDisk: true # enable index node build disk vector index + maxDiskUsagePercentage: 95 + ip: # if not specified, use the first unicastable address + port: 21121 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +dataCoord: + channel: + watchTimeoutInterval: 300 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer. + balanceWithRpc: true # Whether to enable balance with RPC, default to use etcd watch + legacyVersionWithoutRPCWatch: 2.4.0 # Datanodes <= this version are considered as legacy nodes, which doesn't have rpc based watch(). This is only used during rolling upgrade where legacy nodes won't get new channels + balanceSilentDuration: 300 # The duration after which the channel manager start background channel balancing + balanceInterval: 360 # The interval with which the channel manager check dml channel balance status + checkInterval: 1 # The interval in seconds with which the channel manager advances channel states + notifyChannelOperationTimeout: 5 # Timeout notifing channel operations (in seconds). + segment: + maxSize: 1024 # Maximum size of a segment in MB + diskSegmentMaxSize: 2048 # Maximun size of a segment in MB for collection which has Disk index + sealProportion: 0.12 + assignmentExpiration: 2000 # The time of the assignment expiration in ms + allocLatestExpireAttempt: 200 # The time attempting to alloc latest lastExpire from rootCoord after restart + maxLife: 86400 # The max lifetime of segment in seconds, 24*60*60 + # If a segment didn't accept dml records in maxIdleTime and the size of segment is greater than + # minSizeFromIdleToSealed, Milvus will automatically seal it. + # The max idle time of segment in seconds, 10*60. + maxIdleTime: 600 + minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed. + # The max number of binlog file for one segment, the segment will be sealed if + # the number of binlog file reaches to max value. + maxBinlogFileNumber: 32 + smallProportion: 0.5 # The segment is considered as "small segment" when its # of rows is smaller than + # (smallProportion * segment max # of rows). + # A compaction will happen on small segments if the segment after compaction will have + compactableProportion: 0.85 + # over (compactableProportion * segment max # of rows) rows. + # MUST BE GREATER THAN OR EQUAL TO !!! + # During compaction, the size of segment # of rows is able to exceed segment max # of rows by (expansionRate-1) * 100%. + expansionRate: 1.25 + autoUpgradeSegmentIndex: false # whether auto upgrade segment index to index engine's version + enableCompaction: true # Enable data segment compaction + compaction: + enableAutoCompaction: true + indexBasedCompaction: true + rpcTimeout: 10 + maxParallelTaskNum: 10 + workerMaxParallelTaskNum: 2 + levelzero: + forceTrigger: + minSize: 8388608 # The minmum size in bytes to force trigger a LevelZero Compaction, default as 8MB + maxSize: 67108864 # The maxmum size in bytes to force trigger a LevelZero Compaction, default as 64MB + deltalogMinNum: 10 # The minimum number of deltalog files to force trigger a LevelZero Compaction + deltalogMaxNum: 30 # The maxmum number of deltalog files to force trigger a LevelZero Compaction, default as 30 + enableGarbageCollection: true + gc: + interval: 3600 # gc interval in seconds + missingTolerance: 86400 # file meta missing tolerance duration in seconds, default to 24hr(1d) + dropTolerance: 10800 # file belongs to dropped entity tolerance duration in seconds. 3600 + removeConcurrent: 32 # number of concurrent goroutines to remove dropped s3 objects + scanInterval: 168 # garbage collection scan residue interval in hours + enableActiveStandby: false + brokerTimeout: 5000 # 5000ms, dataCoord broker rpc timeout + autoBalance: true # Enable auto balance + checkAutoBalanceConfigInterval: 10 # the interval of check auto balance config + import: + filesPerPreImportTask: 2 # The maximum number of files allowed per pre-import task. + taskRetention: 10800 # The retention period in seconds for tasks in the Completed or Failed state. + maxSizeInMBPerImportTask: 6144 # To prevent generating of small segments, we will re-group imported files. This parameter represents the sum of file sizes in each group (each ImportTask). + scheduleInterval: 2 # The interval for scheduling import, measured in seconds. + checkIntervalHigh: 2 # The interval for checking import, measured in seconds, is set to a high frequency for the import checker. + checkIntervalLow: 120 # The interval for checking import, measured in seconds, is set to a low frequency for the import checker. + maxImportFileNumPerReq: 1024 # The maximum number of files allowed per single import request. + waitForIndex: true # Indicates whether the import operation waits for the completion of index building. + gracefulStopTimeout: 5 # seconds. force stop node without graceful stop + ip: # if not specified, use the first unicastable address + port: 13333 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +dataNode: + dataSync: + flowGraph: + maxQueueLength: 16 # Maximum length of task queue in flowgraph + maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph + maxParallelSyncMgrTasks: 256 # The max concurrent sync task number of datanode sync mgr globally + skipMode: + enable: true # Support skip some timetick message to reduce CPU usage + skipNum: 4 # Consume one for every n records skipped + coldTime: 60 # Turn on skip mode after there are only timetick msg for x seconds + segment: + insertBufSize: 16777216 # Max buffer size to flush for a single segment. + deleteBufBytes: 16777216 # Max buffer size in bytes to flush del for a single channel, default as 16MB + syncPeriod: 600 # The period to sync segments if buffer is not empty. + memory: + forceSyncEnable: true # Set true to force sync if memory usage is too high + forceSyncSegmentNum: 1 # number of segments to sync, segments with top largest buffer will be synced. + checkInterval: 3000 # the interal to check datanode memory usage, in milliseconds + forceSyncWatermark: 0.5 # memory watermark for standalone, upon reaching this watermark, segments will be synced. + timetick: + byRPC: true + interval: 500 + channel: + # specify the size of global work pool of all channels + # if this parameter <= 0, will set it as the maximum number of CPUs that can be executing + # suggest to set it bigger on large collection numbers to avoid blocking + workPoolSize: -1 + # specify the size of global work pool for channel checkpoint updating + # if this parameter <= 0, will set it as 10 + updateChannelCheckpointMaxParallel: 10 + updateChannelCheckpointInterval: 60 # the interval duration(in seconds) for datanode to update channel checkpoint of each channel + updateChannelCheckpointRPCTimeout: 20 # timeout in seconds for UpdateChannelCheckpoint RPC call + maxChannelCheckpointsPerPRC: 128 # The maximum number of channel checkpoints per UpdateChannelCheckpoint RPC. + channelCheckpointUpdateTickInSeconds: 10 # The frequency, in seconds, at which the channel checkpoint updater executes updates. + import: + maxConcurrentTaskNum: 16 # The maximum number of import/pre-import tasks allowed to run concurrently on a datanode. + maxImportFileSizeInGB: 16 # The maximum file size (in GB) for an import file, where an import file refers to either a Row-Based file or a set of Column-Based files. + readBufferSizeInMB: 16 # The data block size (in MB) read from chunk manager by the datanode during import. + compaction: + levelZeroBatchMemoryRatio: 0.05 # The minimal memory ratio of free memory for level zero compaction executing in batch mode + gracefulStopTimeout: 1800 # seconds. force stop node without graceful stop + ip: # if not specified, use the first unicastable address + port: 21124 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + slot: + slotCap: 2 # The maximum number of tasks(e.g. compaction, importing) allowed to run concurrently on a datanode. + +# Configures the system log output. +log: + level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + file: + rootPath: # root dir path to put logs, default "" means no log file will print. please adjust in embedded Milvus: /tmp/milvus/logs + maxSize: 300 # MB + maxAge: 10 # Maximum time for log retention in day. + maxBackups: 20 + format: text # text or json + stdout: true # Stdout enable or not + +grpc: + log: + level: WARNING + gracefulStopTimeout: 10 # second, time to wait graceful stop finish + client: + compressionEnabled: false + dialTimeout: 200 + keepAliveTime: 10000 + keepAliveTimeout: 20000 + maxMaxAttempts: 10 + initialBackoff: 0.2 + maxBackoff: 10 + minResetInterval: 1000 + maxCancelError: 32 + minSessionCheckInterval: 200 + +# # Configure the proxy tls enable. +# tls: +# serverPemPath: configs/cert/server.pem +# serverKeyPath: configs/cert/server.key +# caPemPath: configs/cert/ca.pem + +common: + defaultPartitionName: _default # default partition name for a collection + defaultIndexName: _default_idx # default index name + entityExpiration: -1 # Entity expiration in seconds, CAUTION -1 means never expire + indexSliceSize: 16 # MB + threadCoreCoefficient: + highPriority: 10 # This parameter specify how many times the number of threads is the number of cores in high priority pool + middlePriority: 5 # This parameter specify how many times the number of threads is the number of cores in middle priority pool + lowPriority: 1 # This parameter specify how many times the number of threads is the number of cores in low priority pool + buildIndexThreadPoolRatio: 0.75 + DiskIndex: + MaxDegree: 56 + SearchListSize: 100 + PQCodeBudgetGBRatio: 0.125 + BuildNumThreadsRatio: 1 + SearchCacheBudgetGBRatio: 0.1 + LoadNumThreadRatio: 8 + BeamWidthRatio: 4 + gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency. + gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time. + storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead + # Default value: auto + # Valid values: [auto, avx512, avx2, avx, sse4_2] + # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building. + simdType: auto + security: + authorizationEnabled: false + # The superusers will ignore some system check processes, + # like the old password verification when updating the credential + superUsers: + tlsMode: 0 + session: + ttl: 30 # ttl value when session granting a lease to register service + retryTimes: 30 # retry times when session sending etcd requests + locks: + metrics: + enable: false # whether gather statistics for metrics locks + threshold: + info: 500 # minimum milliseconds for printing durations in info level + warn: 1000 # minimum milliseconds for printing durations in warn level + storage: + scheme: s3 + enablev2: false + ttMsgEnabled: true # Whether the instance disable sending ts messages + traceLogMode: 0 # trace request info + bloomFilterSize: 100000 # bloom filter initial size + maxBloomFalsePositive: 0.001 # max false positive rate for bloom filter + +# QuotaConfig, configurations of Milvus quota and limits. +# By default, we enable: +# 1. TT protection; +# 2. Memory protection. +# 3. Disk quota protection. +# You can enable: +# 1. DML throughput limitation; +# 2. DDL, DQL qps/rps limitation; +# 3. DQL Queue length/latency protection; +# 4. DQL result rate protection; +# If necessary, you can also manually force to deny RW requests. +quotaAndLimits: + enabled: true # `true` to enable quota and limits, `false` to disable. + # quotaCenterCollectInterval is the time interval that quotaCenter + # collects metrics from Proxies, Query cluster and Data cluster. + # seconds, (0 ~ 65536) + quotaCenterCollectInterval: 3 + limits: + allocRetryTimes: 15 # retry times when delete alloc forward data from rate limit failed + allocWaitInterval: 1000 # retry wait duration when delete alloc forward data rate failed, in millisecond + complexDeleteLimitEnable: false # whether complex delete check forward data by limiter + maxCollectionNum: 65536 + maxCollectionNumPerDB: 65536 + maxInsertSize: -1 # maximum size of a single insert request, in bytes, -1 means no limit + maxResourceGroupNumOfQueryNode: 1024 # maximum number of resource groups of query nodes + ddl: + enabled: false + collectionRate: -1 # qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection + partitionRate: -1 # qps, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition + db: + collectionRate: -1 # qps of db level , default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection + partitionRate: -1 # qps of db level, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition + indexRate: + enabled: false + max: -1 # qps, default no limit, rate for CreateIndex, DropIndex + db: + max: -1 # qps of db level, default no limit, rate for CreateIndex, DropIndex + flushRate: + enabled: false + max: -1 # qps, default no limit, rate for flush + collection: + max: -1 # qps, default no limit, rate for flush at collection level. + db: + max: -1 # qps of db level, default no limit, rate for flush + compactionRate: + enabled: false + max: -1 # qps, default no limit, rate for manualCompaction + db: + max: -1 # qps of db level, default no limit, rate for manualCompaction + dml: + # dml limit rates, default no limit. + # The maximum rate will not be greater than max. + enabled: false + insertRate: + max: -1 # MB/s, default no limit + db: + max: -1 # MB/s, default no limit + collection: + max: -1 # MB/s, default no limit + partition: + max: -1 # MB/s, default no limit + upsertRate: + max: -1 # MB/s, default no limit + db: + max: -1 # MB/s, default no limit + collection: + max: -1 # MB/s, default no limit + partition: + max: -1 # MB/s, default no limit + deleteRate: + max: -1 # MB/s, default no limit + db: + max: -1 # MB/s, default no limit + collection: + max: -1 # MB/s, default no limit + partition: + max: -1 # MB/s, default no limit + bulkLoadRate: + max: -1 # MB/s, default no limit, not support yet. TODO: limit bulkLoad rate + db: + max: -1 # MB/s, default no limit, not support yet. TODO: limit db bulkLoad rate + collection: + max: -1 # MB/s, default no limit, not support yet. TODO: limit collection bulkLoad rate + partition: + max: -1 # MB/s, default no limit, not support yet. TODO: limit partition bulkLoad rate + dql: + # dql limit rates, default no limit. + # The maximum rate will not be greater than max. + enabled: false + searchRate: + max: -1 # vps (vectors per second), default no limit + db: + max: -1 # vps (vectors per second), default no limit + collection: + max: -1 # vps (vectors per second), default no limit + partition: + max: -1 # vps (vectors per second), default no limit + queryRate: + max: -1 # qps, default no limit + db: + max: -1 # qps, default no limit + collection: + max: -1 # qps, default no limit + partition: + max: -1 # qps, default no limit + limitWriting: + # forceDeny false means dml requests are allowed (except for some + # specific conditions, such as memory of nodes to water marker), true means always reject all dml requests. + forceDeny: false + ttProtection: + enabled: false + # maxTimeTickDelay indicates the backpressure for DML Operations. + # DML rates would be reduced according to the ratio of time tick delay to maxTimeTickDelay, + # if time tick delay is greater than maxTimeTickDelay, all DML requests would be rejected. + # seconds + maxTimeTickDelay: 300 + memProtection: + # When memory usage > memoryHighWaterLevel, all dml requests would be rejected; + # When memoryLowWaterLevel < memory usage < memoryHighWaterLevel, reduce the dml rate; + # When memory usage < memoryLowWaterLevel, no action. + enabled: true + dataNodeMemoryLowWaterLevel: 0.85 # (0, 1], memoryLowWaterLevel in DataNodes + dataNodeMemoryHighWaterLevel: 0.95 # (0, 1], memoryHighWaterLevel in DataNodes + queryNodeMemoryLowWaterLevel: 0.85 # (0, 1], memoryLowWaterLevel in QueryNodes + queryNodeMemoryHighWaterLevel: 0.95 # (0, 1], memoryHighWaterLevel in QueryNodes + growingSegmentsSizeProtection: + # No action will be taken if the growing segments size is less than the low watermark. + # When the growing segments size exceeds the low watermark, the dml rate will be reduced, + # but the rate will not be lower than minRateRatio * dmlRate. + enabled: false + minRateRatio: 0.5 + lowWaterLevel: 0.2 + highWaterLevel: 0.4 + diskProtection: + enabled: true # When the total file size of object storage is greater than `diskQuota`, all dml requests would be rejected; + diskQuota: -1 # MB, (0, +inf), default no limit + diskQuotaPerDB: -1 # MB, (0, +inf), default no limit + diskQuotaPerCollection: -1 # MB, (0, +inf), default no limit + diskQuotaPerPartition: -1 # MB, (0, +inf), default no limit + limitReading: + # forceDeny false means dql requests are allowed (except for some + # specific conditions, such as collection has been dropped), true means always reject all dql requests. + forceDeny: false + queueProtection: + enabled: false + # nqInQueueThreshold indicated that the system was under backpressure for Search/Query path. + # If NQ in any QueryNode's queue is greater than nqInQueueThreshold, search&query rates would gradually cool off + # until the NQ in queue no longer exceeds nqInQueueThreshold. We think of the NQ of query request as 1. + # int, default no limit + nqInQueueThreshold: -1 + # queueLatencyThreshold indicated that the system was under backpressure for Search/Query path. + # If dql latency of queuing is greater than queueLatencyThreshold, search&query rates would gradually cool off + # until the latency of queuing no longer exceeds queueLatencyThreshold. + # The latency here refers to the averaged latency over a period of time. + # milliseconds, default no limit + queueLatencyThreshold: -1 + resultProtection: + enabled: false + # maxReadResultRate indicated that the system was under backpressure for Search/Query path. + # If dql result rate is greater than maxReadResultRate, search&query rates would gradually cool off + # until the read result rate no longer exceeds maxReadResultRate. + # MB/s, default no limit + maxReadResultRate: -1 + maxReadResultRatePerDB: -1 + maxReadResultRatePerCollection: -1 + # colOffSpeed is the speed of search&query rates cool off. + # (0, 1] + coolOffSpeed: 0.9 + +trace: + # trace exporter type, default is stdout, + # optional values: ['noop','stdout', 'jaeger', 'otlp'] + exporter: noop + # fraction of traceID based sampler, + # optional values: [0, 1] + # Fractions >= 1 will always sample. Fractions < 0 are treated as zero. + sampleFraction: 0 + jaeger: + url: # when exporter is jaeger should set the jaeger's URL + otlp: + endpoint: # example: "127.0.0.1:4318" + secure: true + +#when using GPU indexing, Milvus will utilize a memory pool to avoid frequent memory allocation and deallocation. +#here, you can set the size of the memory occupied by the memory pool, with the unit being MB. +#note that there is a possibility of Milvus crashing when the actual memory demand exceeds the value set by maxMemSize. +#if initMemSize and MaxMemSize both set zero, +#milvus will automatically initialize half of the available GPU memory, +#maxMemSize will the whole available GPU memory. +gpu: + initMemSize: # Gpu Memory Pool init size + maxMemSize: # Gpu Memory Pool Max size \ No newline at end of file diff --git a/src/config/minio_cli.env b/src/config/minio_cli.env new file mode 100644 index 00000000..f7c629a4 --- /dev/null +++ b/src/config/minio_cli.env @@ -0,0 +1,7 @@ +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_ACCESS_KEY=minioadmin +MINIO_USE_SSL=false +MINIO_BUCKET_NAME=documents +MINIO_REGION=local +MINIO_ENDPOINT=minio:9000 +MINIO_MOCKED=false diff --git a/src/config/nats_cli.env b/src/config/nats_cli.env new file mode 100644 index 00000000..e8ba47c0 --- /dev/null +++ b/src/config/nats_cli.env @@ -0,0 +1,24 @@ +NATS_CLIENT_URL=nats://nats:4222 +# TODO: ANDRII TO IMPLEMENT TIMEOUTS IN CONNECTOR +# TODO: ANDRII TO IMPLEMENT STREAM_NAME, SUBJECT, ACK_WAIT, MAX_DELIVER IN CONNECTOR +NATS_CLIENT_CONNECT_TIMEOUT=300 +NATS_CLIENT_RECONNECT_TIME_WAIT=30 +NATS_CLIENT_MAX_RECONNECT_ATTEMPTS=3 + +NATS_CLIENT_SEMANTIC_STREAM_NAME=semantic +NATS_CLIENT_SEMANTIC_STREAM_SUBJECT=semantic_activity +NATS_CLIENT_SEMANTIC_ACK_WAIT=14400 +NATS_CLIENT_SEMANTIC_MAX_DELIVER=3 + +NATS_CLIENT_WHISPER_STREAM_NAME=whisper +NATS_CLIENT_WHISPER_STREAM_SUBJECT=whisper_activity +NATS_CLIENT_WHISPER_ACK_WAIT=14400 +NATS_CLIENT_WHISPER_MAX_DELIVER=3 + +NATS_CLIENT_CONNECTOR_STREAM_NAME=connector +NATS_CLIENT_CONNECTOR_STREAM_SUBJECT=connector_trigger +NATS_CLIENT_CONNECTOR_ACK_WAIT=30 # in seconds +NATS_CLIENT_CONNECTOR_MAX_DELIVER=3 + + + diff --git a/src/config/nats_config.json b/src/config/nats_config.json new file mode 100644 index 00000000..b35687e1 --- /dev/null +++ b/src/config/nats_config.json @@ -0,0 +1,7 @@ +{ + "server": { + "name": "nats", + "url": "http://localhost:8000/proxy/" + }, + "hideServerInput": true +} \ No newline at end of file diff --git a/src/config/oauth_cli.env b/src/config/oauth_cli.env new file mode 100644 index 00000000..4aff2128 --- /dev/null +++ b/src/config/oauth_cli.env @@ -0,0 +1 @@ +OAUTH_URL=http://api:8080 \ No newline at end of file diff --git a/src/config/orchestrator_srv.env b/src/config/orchestrator_srv.env new file mode 100644 index 00000000..27b39fec --- /dev/null +++ b/src/config/orchestrator_srv.env @@ -0,0 +1,2 @@ +FILE_SIZE_LIMIT=1 # GB +ORCHESTRATOR_RENEW_INTERVAL=30 # seconds \ No newline at end of file diff --git a/src/config/prometheus.yml b/src/config/prometheus.yml new file mode 100644 index 00000000..9d7074cd --- /dev/null +++ b/src/config/prometheus.yml @@ -0,0 +1,84 @@ +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - static_configs: + - targets: [ ] + scheme: http + timeout: 10s + api_version: v1 + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + # external_labels: + # monitor: 'cognix-monitor' + +# A scrape configuration containing one endpoint to scrape: +# Here it's Prometheus itself. + +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. +# - job_name: 'prometheus' +# # Override the global default and scrape targets from this job every 5 seconds. +# scrape_interval: 5s +# static_configs: +# - targets: ['localhost:9090'] + +## disabled to get host system logs +# - job_name: system +# static_configs: +# - targets: +# - localhost +# labels: +# job: varlogs +# __path__: /var/log/*log + + # Example job for node_exporter + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: 'nats-exporter' + static_configs: + - targets: [ 'nats-exporter:7777' ] + + - job_name: 'milvus-standalone' + static_configs: + - targets: [ 'milvus-standalone:9091' ] + + - job_name: 'prometheus' + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - localhost:9090 + - + - job_name: 'cockroachdb' + metrics_path: '/_status/vars' + # Insecure mode: + scheme: 'http' + # Secure mode: + # scheme: 'https' + tls_config: + insecure_skip_verify: true + static_configs: + - targets: ['cockroach:8080'] + labels: + cluster: 'my-cockroachdb-cluster' + + - job_name: docker + static_configs: + - targets: + - localhost + labels: + job: docker_logs + __path__: /var/lib/docker/containers/*/*-json.log \ No newline at end of file diff --git a/src/config/promtail-config.yaml b/src/config/promtail-config.yaml new file mode 100644 index 00000000..ed06e8cd --- /dev/null +++ b/src/config/promtail-config.yaml @@ -0,0 +1,18 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: +- job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log diff --git a/src/config/promtail-local-config.yaml b/src/config/promtail-local-config.yaml new file mode 100644 index 00000000..deb91e0f --- /dev/null +++ b/src/config/promtail-local-config.yaml @@ -0,0 +1,31 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: +# enable if you want to collect host system logs +#- job_name: system +# static_configs: +# - targets: +# - localhost +# labels: +# job: varlogs +# __path__: /var/log/*log +# stream: stdout + + - job_name: 'docker' + docker_sd_configs: + - host: unix:///var/run/docker.sock + relabel_configs: + - source_labels: [__meta_docker_container_name] + target_label: job + - source_labels: [__meta_docker_container_name] + target_label: instance + pipeline_stages: + - docker: {} \ No newline at end of file diff --git a/src/config/semantic_srv.env b/src/config/semantic_srv.env new file mode 100644 index 00000000..e67d3011 --- /dev/null +++ b/src/config/semantic_srv.env @@ -0,0 +1,14 @@ +LOG_LEVEL=INFO +LOG_FORMAT='%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(message)s' +LOCAL_TEMP_PATH=../temp + +# used for RecursiveCharacterTextSplitter. Set custom chunk size and overlap +CHUNK_SIZE=500 +CHUNK_OVERLAP=20 + +# in seconds +# used on the internal readiness probe +# internally, the code will notify the readiness service that it is alive +# if the readiness service does not receive notification, after the amount of time +# in second specified in this var, it will reply to any healthz request with 503 +READINESS_TIME_OUT=3600 \ No newline at end of file diff --git a/src/config/web_srv.env b/src/config/web_srv.env new file mode 100644 index 00000000..36920be0 --- /dev/null +++ b/src/config/web_srv.env @@ -0,0 +1,21 @@ +VITE_PLATFORM_API_URL= +VITE_PLATFORM_API_LOGIN_URL="/api/auth/google/login" +VITE_PLATFORM_API_LOGIN_CALLBACK_URL="/api/auth/google/callback" +VITE_PLATFORM_API_CHAT_LIST_URL="/api/chats/get-user-chat-sessions" +VITE_PLATFORM_API_CHAT_CREATE_URL="/api/chats/create-chat-session" +VITE_PLATFORM_API_CHAT_DETAIL_URL="/api/chats/get-chat-session" +VITE_PLATFORM_API_USER_INFO_URL="/api/tenant/user_info" +VITE_PLATFORM_API_USER_CHATS_URL="/api/chats/get-user-chat-sessions" +VITE_PLATFORM_API_CONNECTOR_CREATE_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_EDIT_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_LIST_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_DETAIL_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_DELETE_URL="/api/manage/connector" +VITE_PLATFORM_API_CHAT_SEND_MESSAGE_URL="/api/chats/send-message" +VITE_PLATFORM_API_CHAT_FEEDBACK_MESSAGE_URL="/api/chats/message/feedback" +VITE_PLATFORM_API_LLM_LIST_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_CREATE_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_DETAIL_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_EDIT_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_DELETE_URL="/api/manage/personas" +VITE_PLATFORM_API_SOURCE_TYPES_LIST_URL="/api/manage/connector/source_types" diff --git a/src/config/yugabytedb_cli.env b/src/config/yugabytedb_cli.env new file mode 100644 index 00000000..9bbe24cf --- /dev/null +++ b/src/config/yugabytedb_cli.env @@ -0,0 +1,6 @@ +# would be better to name the variable with POSTGRES as it should work with any PostgreSQL compatible database +# +COCKROACH_CLIENT_DATABASE_URL_GOLANG=postgresql://yugabyte:yugabyte@yugabytedb:5433/yugabyte?sslmode=disable +COCKROACH_CLIENT_DATABASE_URL='postgres://yugabyte:yugabyte@yugabytedb:5433/yugabyte?sslmode=disable' +COCKROACH_CLIENT_DB_DEBUG=false + diff --git a/src/db-schema/schema.sql b/src/db-schema/schema.sql new file mode 100644 index 00000000..8ab2bb8c --- /dev/null +++ b/src/db-schema/schema.sql @@ -0,0 +1,148 @@ +create table tenants +( + id uuid not null primary key, + name varchar(255), + configuration jsonb default '{}'::JSONB not null +); + +create table users +( + id uuid not null primary key, + tenant_id uuid not null references tenants, + user_name varchar(255) not null unique, + first_name varchar(255), + last_name varchar(255), + external_id text, + roles text[] default '{}'::STRING[] not null +); + +create table llms +( + id bigint default unique_rowid() not null primary key, + tenant_id uuid not null references tenants, + name varchar(255) not null, + model_id varchar(255) not null, + url varchar not null, + api_key varchar, + endpoint varchar, + creation_date timestamp not null, + last_update timestamp, + deleted_date timestamp +); + +create table embedding_models +( + id bigint default unique_rowid() not null primary key, + tenant_id uuid not null, + model_id varchar not null, + model_name varchar not null, + model_dim bigint not null, + url varchar, + is_active boolean default false not null, + creation_date timestamp not null, + last_update timestamp, + deleted_date timestamp +); + +create table personas +( + id bigint default unique_rowid() not null primary key, + name varchar not null, + llm_id bigint references llm, + default_persona boolean not null, + description varchar not null, + tenant_id uuid not null references tenants, + is_visible boolean not null, + display_priority bigint, + starter_messages jsonb default '{}'::JSONB not null, + creation_date timestamp not null, + last_update timestamp, + deleted_date timestamp +); + +create table prompts +( + id bigint default unique_rowid() not null primary key, + persona_id bigint not null references personas, + user_id uuid not null references users, + name varchar not null, + description varchar not null, + system_prompt text not null, + task_prompt text not null, + creation_date timestamp not null, + last_update timestamp, + deleted_date timestamp +); + +create table connectors +( + id bigint default unique_rowid() not null primary key, + name varchar not null, + type varchar(50) not null, + connector_specific_config jsonb not null, + state jsonb not null default '{}'::jsonb, + refresh_freq bigint, + user_id uuid not null references public.users, + tenant_id uuid references public.tenants, + last_successful_analyzed timestamp, + status varchar, + total_docs_analyzed bigint not null, + creation_date timestamp not null, + last_update timestamp, + deleted_date timestamp +); + +create table chat_sessions +( + id bigint default unique_rowid() not null primary key, + user_id uuid not null references users, + description text not null, + creation_date timestamp not null, + deleted_date timestamp, + persona_id bigint not null references personas, + one_shot boolean not null +); + +create table chat_messages +( + id bigint default unique_rowid() not null primary key, + chat_session_id bigint not null references chat_sessions, + message text not null, + message_type varchar(9) not null, + time_sent timestamp not null, + token_count bigint not null, + parent_message bigint, + latest_child_message bigint, + rephrased_query text, + error text +); + +create table chat_message_feedbacks +( + id bigint default unique_rowid() not null primary key, + chat_message_id bigint not null references chat_messages, + user_id uuid not null references users, + up_votes boolean not null, + feedback varchar default ''::STRING not null +); + +CREATE TABLE documents ( + id SERIAL PRIMARY KEY NOT NULL, + parent_id bigint REFERENCES documents(id), -- Allows nulls, used for URLs + connector_id bigint NOT NULL REFERENCES connectors(id), + source_id text NOT NULL, -- unique id from source url for web, id for other services + url text, -- url for web connector, link (minio:bucket:file) for file in minio + signature text, + chunking_session uuid, -- Allows nulls + analyzed bool NOT NULL DEFAULT FALSE, -- default false, true when semantic created the embeddings in the vector db + creation_date timestamp WITHOUT TIME ZONE NOT NULL, --datetime utc IMPORTANT now() will not get the utc date!!!! + last_update timestamp WITHOUT TIME ZONE, + original_url text +); + +create table chat_message_document_pairs +( + id bigint default unique_rowid() not null primary key, + chat_message_id bigint not null references public.chat_messages on delete cascade, + document_id bigint not null references public.documents on delete cascade +); diff --git a/src/main.tsx b/src/main.tsx deleted file mode 100644 index 7ef379a6..00000000 --- a/src/main.tsx +++ /dev/null @@ -1,121 +0,0 @@ -import React from "react"; -import ReactDOM from "react-dom/client"; -import { RouterProvider, createBrowserRouter } from "react-router-dom"; -// import "@/global.css"; -// import "@fontsource-variable/inter"; - -const router = createBrowserRouter([ - { - path: "/", - children: [ - // { - // path: "login", - // children: [ - // { - // path: "sign-in", - // lazy: () => import("@/pages/login/sign-in"), - // }, - // { - // path: "sign-up", - // lazy: () => import("@/pages/login/sign-up"), - // }, - // { - // path: "onboarding", - // lazy: () => import("@/pages/login/onboarding"), - // }, - // { - // path: "user", - // lazy: () => import("@/pages/login/user"), - // }, - // ], - // }, - { - path: "chat", - lazy: () => import("@/pages/chat"), - }, - { - path: "settings", - children: [ - { - path: "connectors", - children: [ - { - path: "existing-connectors", - lazy: () => - import("@/pages/settings/connectors/existing-connectors"), - }, - { - path: "add-connector", - lazy: () => import("@/pages/settings/connectors/add-connector"), - }, - ], - }, - { - path: "documents", - children: [ - { - path: "document-sets", - lazy: () => import("@/pages/settings/documents/sets"), - }, - { - path: "explorer", - lazy: () => import("@/pages/settings/documents/explorer"), - }, - { - path: "feedback", - lazy: () => import("@/pages/settings/documents/feedback"), - }, - ], - }, - { - path: "custom-assistant", - children: [ - { - path: "personas", - lazy: () => - import("@/pages/settings/custom-assistant/personas"), - }, - { - path: "slack-bots", - lazy: () => - import("@/pages/settings/custom-assistant/slack-bots"), - }, - { - path: "teams", - lazy: () => import("@/pages/settings/custom-assistant/teams"), - }, - ], - }, - { - path: "model-config", - children: [ - { - path: "llms", - lazy: () => import("@/pages/settings/model-config/llms"), - }, - { - path: "embedding", - lazy: () => import("@/pages/settings/model-config/embedding"), - }, - ], - }, - { - path: "user-management", - children: [ - { - path: "users", - lazy: () => import("@/pages/settings/user-management/users"), - }, - ], - }, - ], - }, - ], - }, -]); - -ReactDOM.createRoot(document.getElementById("root")!).render( - - - -); diff --git a/src/pages/chat/index.tsx b/src/pages/chat/index.tsx deleted file mode 100644 index c86594a3..00000000 --- a/src/pages/chat/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function ChatComponent() { - return
Chat
; -} - -export { ChatComponent as Component }; diff --git a/src/pages/index.tsx b/src/pages/index.tsx deleted file mode 100644 index c09b8f3e..00000000 --- a/src/pages/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import { Outlet } from 'react-router-dom'; -import ThemeProvider from '@/context/ThemeContext'; -import AuthProvider from '@/context/AuthContext'; -import { Toaster } from '@/components/ui/sonner'; -import NotificationProvider from '@/context/NotificationContext'; -import { useMixpanelTracking } from '@/lib/hooks/use-mixpanel-tracking'; - -export function Providers() { - useMixpanelTracking(); - - return ( - - - - - - - - - ); -} - -export { Providers as Component }; diff --git a/src/pages/login/index.tsx b/src/pages/login/index.tsx deleted file mode 100644 index 37c749a4..00000000 --- a/src/pages/login/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function LoginComponent() { - return
Login
; -} - -export { LoginComponent as Component }; diff --git a/src/pages/settings/connectors/existing-connectors/index.tsx b/src/pages/settings/connectors/existing-connectors/index.tsx deleted file mode 100644 index 0ca81ae9..00000000 --- a/src/pages/settings/connectors/existing-connectors/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
connectors existing
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/custom-assistant/index.tsx b/src/pages/settings/custom-assistant/index.tsx deleted file mode 100644 index edd3510c..00000000 --- a/src/pages/settings/custom-assistant/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function CustomAssistantComponent() { - return
Custom-Assistant
; -} - -export { CustomAssistantComponent as Component }; diff --git a/src/pages/settings/custom-assistant/personas/index.tsx b/src/pages/settings/custom-assistant/personas/index.tsx deleted file mode 100644 index bd26479c..00000000 --- a/src/pages/settings/custom-assistant/personas/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
custom-assistant personas
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/custom-assistant/slack-bots/index.tsx b/src/pages/settings/custom-assistant/slack-bots/index.tsx deleted file mode 100644 index 0981169e..00000000 --- a/src/pages/settings/custom-assistant/slack-bots/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
custom-assistant slack-bots
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/custom-assistant/teams/index.tsx b/src/pages/settings/custom-assistant/teams/index.tsx deleted file mode 100644 index d3e8e7fc..00000000 --- a/src/pages/settings/custom-assistant/teams/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
custom-assistant teams
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/documents/explorer/index.tsx b/src/pages/settings/documents/explorer/index.tsx deleted file mode 100644 index 1134e8d2..00000000 --- a/src/pages/settings/documents/explorer/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
documents explorer
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/documents/feedback/index.tsx b/src/pages/settings/documents/feedback/index.tsx deleted file mode 100644 index 8a757dc9..00000000 --- a/src/pages/settings/documents/feedback/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
documents feedback
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/documents/index.tsx b/src/pages/settings/documents/index.tsx deleted file mode 100644 index 9cba07d5..00000000 --- a/src/pages/settings/documents/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function DocumentComponent() { - return
Documents
; -} - -export { DocumentComponent as Component }; diff --git a/src/pages/settings/model-config/embedding/index.tsx b/src/pages/settings/model-config/embedding/index.tsx deleted file mode 100644 index e6c14f62..00000000 --- a/src/pages/settings/model-config/embedding/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
model-config embedding
; -} - -export { UserManagementComponent as Component }; diff --git a/src/pages/settings/model-config/index.tsx b/src/pages/settings/model-config/index.tsx deleted file mode 100644 index d4f4a756..00000000 --- a/src/pages/settings/model-config/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function ModelConfigComponent() { - return
Model config
; -} - -export { ModelConfigComponent as Component }; diff --git a/src/pages/settings/model-config/llms/index.tsx b/src/pages/settings/model-config/llms/index.tsx deleted file mode 100644 index 5c3cb0de..00000000 --- a/src/pages/settings/model-config/llms/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export function UserManagementComponent() { - return
model-config LLMs
; -} - -export { UserManagementComponent as Component }; diff --git a/src/proto/.DS_Store b/src/proto/.DS_Store new file mode 100644 index 00000000..5008ddfc Binary files /dev/null and b/src/proto/.DS_Store differ diff --git a/src/proto/Makefile b/src/proto/Makefile new file mode 100644 index 00000000..accb366b --- /dev/null +++ b/src/proto/Makefile @@ -0,0 +1,11 @@ + +gen-proto-go: + + protoc -I=. -I=vendor -I=${GOPATH}/src file_type.proto whisper_data.proto embed_service.proto connector_messages.proto semantic_data.proto --go_out=.. --go-grpc_out=.. + + +#PROTOC_GEN_GRPC_PYTHON=$(shell which grpc_tools.protoc) +#.PHONY: gen-proto-python +#gen-proto-python: +# rm -f ./python/embedder/*pb2*.py* +# python -I=. --proto_path=. --python_out=./python --pyi_out=./python --grpc_python_out=./python semantic_data.proto diff --git a/src/proto/connector_messages.proto b/src/proto/connector_messages.proto new file mode 100644 index 00000000..77aae823 --- /dev/null +++ b/src/proto/connector_messages.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package com.cognix; +option go_package = "backend/core/proto;proto"; + + +message ConnectorRequest { + int64 id = 1; + map params = 2; +} + +message EmbeddAsyncRequest { + int64 document_id = 1; + int64 chunk_id = 2; + string content = 3; + string model_id = 4; +} + +message EmbeddAsyncResponse { + int64 document_id = 1; + int64 chunk_id = 2; + string content = 3; + repeated float vector = 4; +} \ No newline at end of file diff --git a/src/proto/embed_service.proto b/src/proto/embed_service.proto new file mode 100644 index 00000000..5764001c --- /dev/null +++ b/src/proto/embed_service.proto @@ -0,0 +1,18 @@ +// gRPC Service +syntax = "proto3"; + +package com.cognix; +option go_package = "backend/core/proto;proto"; + +message EmbedRequest { + string content = 1; + string model = 2; +} + +message EmbedResponse { + repeated float vector = 1; +} + +service EmbedService { + rpc GetEmbeding (EmbedRequest) returns (EmbedResponse) {} +} diff --git a/src/proto/file_type.proto b/src/proto/file_type.proto new file mode 100644 index 00000000..5836047e --- /dev/null +++ b/src/proto/file_type.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; +package com.cognix; +option go_package = "backend/core/proto;proto"; +enum FileType { + // ORCHESTRATOR SEND TO CONNECTOR: + UNKNOWN = 0; + PDF = 1; + XPS = 2; + TXT = 3; + DOCX = 4; + DOC = 5; + XLSX = 6; + XLS = 7; + PPTX = 8; + PPT = 9; + HWPX = 10; + EPUB = 11; + MOBI = 12; + FB2 = 13; + CBZ = 14; + MD = 15; + // ORCHESTRATOR SEND TO SEMANTIC + URL = 16; + YT = 17; + + // ORCHESTRATOR SEND TO WHISPER + MP4 = 18; + MP3 = 19; + MPEG = 20; + MPGA = 21; + M4A = 22; + WAV = 23; + WEBM = 24; + MOV = 25; + + // add all supported file that in another document + // check what with Google docs +}; \ No newline at end of file diff --git a/src/proto/semantic_data.proto b/src/proto/semantic_data.proto new file mode 100644 index 00000000..40b5e991 --- /dev/null +++ b/src/proto/semantic_data.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +import "file_type.proto"; +package com.cognix; +option go_package = "backend/core/proto;proto"; + +message SemanticData { + // This is the url where the file is located. + // Based on the chunking type it will be a WEB URL (HTML type) + // Will be an S3/MINIO link with a proper authentication in case of a file + string url = 1; + bool url_recursive = 2; + string site_map = 3; + bool search_for_sitemap = 4; + int64 document_id = 5; + int64 connector_id = 6; + FileType file_type = 7; + string collection_name = 8; + string model_name = 9; + int32 model_dimension = 10; +} diff --git a/src/proto/telemetry_service.proto b/src/proto/telemetry_service.proto new file mode 100644 index 00000000..dd251039 --- /dev/null +++ b/src/proto/telemetry_service.proto @@ -0,0 +1,15 @@ +// gRPC Service +syntax = "proto3"; + +package com.cognix; +option go_package = "backend/core/proto;proto"; + +message TelemetryMessage { + string content = 1; + string model = 2; +} + + +service TelemetryService { + rpc Trace (TelemetryMessage) returns () {} +} diff --git a/src/proto/whisper_data.proto b/src/proto/whisper_data.proto new file mode 100644 index 00000000..9ddb8fad --- /dev/null +++ b/src/proto/whisper_data.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +import "file_type.proto"; +package com.cognix; +option go_package = "backend/core/proto;proto"; + + +message WhisperData { + // This is the url where the file is located. + // Based on the chunking type it will be a WEB URL (HTML type) + // Will be an S3/MINIO link with a proper authentication in case of a file + string url = 1; + int64 document_id = 2; + int64 connector_id = 3; + FileType file_type = 4; + string collection_name = 5; + string model_name = 6; + int32 model_dimension = 7; +} diff --git a/src/telemetry/readme.md b/src/telemetry/readme.md new file mode 100644 index 00000000..cee344ec --- /dev/null +++ b/src/telemetry/readme.md @@ -0,0 +1,12 @@ +This is the API (gRPC) that we are using to receive anonymous data about CogniX + +
+ +If you wnat to disable, just edit the source code and build from your branch +
+ +In the future the will a param that will allow telemetry to be disabled +
+Anyways if you leave it on you''l help us improve :) ' + +GO grpc service diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts deleted file mode 100644 index 11f02fe2..00000000 --- a/src/vite-env.d.ts +++ /dev/null @@ -1 +0,0 @@ -/// diff --git a/src/web/.env.example b/src/web/.env.example new file mode 100644 index 00000000..36920be0 --- /dev/null +++ b/src/web/.env.example @@ -0,0 +1,21 @@ +VITE_PLATFORM_API_URL= +VITE_PLATFORM_API_LOGIN_URL="/api/auth/google/login" +VITE_PLATFORM_API_LOGIN_CALLBACK_URL="/api/auth/google/callback" +VITE_PLATFORM_API_CHAT_LIST_URL="/api/chats/get-user-chat-sessions" +VITE_PLATFORM_API_CHAT_CREATE_URL="/api/chats/create-chat-session" +VITE_PLATFORM_API_CHAT_DETAIL_URL="/api/chats/get-chat-session" +VITE_PLATFORM_API_USER_INFO_URL="/api/tenant/user_info" +VITE_PLATFORM_API_USER_CHATS_URL="/api/chats/get-user-chat-sessions" +VITE_PLATFORM_API_CONNECTOR_CREATE_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_EDIT_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_LIST_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_DETAIL_URL="/api/manage/connector" +VITE_PLATFORM_API_CONNECTOR_DELETE_URL="/api/manage/connector" +VITE_PLATFORM_API_CHAT_SEND_MESSAGE_URL="/api/chats/send-message" +VITE_PLATFORM_API_CHAT_FEEDBACK_MESSAGE_URL="/api/chats/message/feedback" +VITE_PLATFORM_API_LLM_LIST_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_CREATE_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_DETAIL_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_EDIT_URL="/api/manage/personas" +VITE_PLATFORM_API_LLM_DELETE_URL="/api/manage/personas" +VITE_PLATFORM_API_SOURCE_TYPES_LIST_URL="/api/manage/connector/source_types" diff --git a/.eslintrc.cjs b/src/web/.eslintrc.cjs similarity index 72% rename from .eslintrc.cjs rename to src/web/.eslintrc.cjs index d6c95379..e109b92d 100644 --- a/.eslintrc.cjs +++ b/src/web/.eslintrc.cjs @@ -5,8 +5,9 @@ module.exports = { 'eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:react-hooks/recommended', + 'plugin:tailwindcss/recommended', ], - ignorePatterns: ['dist', '.eslintrc.cjs'], + ignorePatterns: ['dist', '.eslintrc.cjs', 'tailwind.config.js'], parser: '@typescript-eslint/parser', plugins: ['react-refresh'], rules: { @@ -14,5 +15,6 @@ module.exports = { 'warn', { allowConstantExport: true }, ], + 'tailwindcss/no-custom-classname': 'off', }, } diff --git a/src/web/.gitignore b/src/web/.gitignore new file mode 100644 index 00000000..438657a9 --- /dev/null +++ b/src/web/.gitignore @@ -0,0 +1,25 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local +.env + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/src/web/Dockerfile b/src/web/Dockerfile new file mode 100644 index 00000000..a89d4e54 --- /dev/null +++ b/src/web/Dockerfile @@ -0,0 +1,32 @@ +# stage1 as builder +FROM node:20.10.0-alpine as builder + +# copy the package.json to install dependencies +COPY package.json package-lock.json ./ + +# Install the dependencies and make the folder +RUN npm install && mkdir /react-ui && mv ./node_modules ./react-ui + +WORKDIR /react-ui + +COPY . . + +COPY .env.example .env + +# Build the project and copy the files +RUN npm run build + + +FROM nginx:alpine3.17 + +#!/bin/sh + +COPY ./nginx/nginx.conf /etc/nginx/nginx.conf + +## Remove default nginx index page +# RUN rm -rf /usr/share/nginx/html/* + +# Copy from the stahg 1 +COPY --from=builder /react-ui/dist /usr/share/nginx/html + +ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/src/web/Dockerfile.cloud b/src/web/Dockerfile.cloud new file mode 100644 index 00000000..2be31f5c --- /dev/null +++ b/src/web/Dockerfile.cloud @@ -0,0 +1,32 @@ +# stage1 as builder +FROM node:20.10.0-alpine as builder + +# copy the package.json to install dependencies +COPY package.json package-lock.json ./ + +# Install the dependencies and make the folder +RUN npm install && mkdir /react-ui && mv ./node_modules ./react-ui + +WORKDIR /react-ui + +COPY . . + +COPY .env.example .env + +# Build the project and copy the files +RUN npm run build + + +FROM nginx:alpine3.17 + +#!/bin/sh + +COPY ./nginx/nginx-cloud.conf /etc/nginx/nginx.conf + +## Remove default nginx index page +# RUN rm -rf /usr/share/nginx/html/* + +# Copy from the stahg 1 +COPY --from=builder /react-ui/dist /usr/share/nginx/html + +ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/src/web/Dockerfile.dev b/src/web/Dockerfile.dev new file mode 100644 index 00000000..9e0eb525 --- /dev/null +++ b/src/web/Dockerfile.dev @@ -0,0 +1,32 @@ +# stage1 as builder +FROM node:20.10.0-alpine as builder + +# copy the package.json to install dependencies +COPY package.json package-lock.json ./ + +# Install the dependencies and make the folder +RUN npm install && mkdir /react-ui && mv ./node_modules ./react-ui + +WORKDIR /react-ui + +COPY . . + +COPY .env.example .env + +# Build the project and copy the files +RUN npm run build + + +FROM nginx:alpine3.17 + +#!/bin/sh + +COPY ./nginx/nginx-dev.conf /etc/nginx/nginx.conf + +## Remove default nginx index page +# RUN rm -rf /usr/share/nginx/html/* + +# Copy from the stahg 1 +COPY --from=builder /react-ui/dist /usr/share/nginx/html + +ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/README.md b/src/web/README.md similarity index 100% rename from README.md rename to src/web/README.md diff --git a/src/web/components.json b/src/web/components.json new file mode 100644 index 00000000..77c60063 --- /dev/null +++ b/src/web/components.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": false, + "tsx": true, + "tailwind": { + "config": "tailwind.config.js", + "css": "src/global.css", + "baseColor": "slate", + "cssVariables": true + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils" + } +} \ No newline at end of file diff --git a/src/web/deploy/kustomization.yaml b/src/web/deploy/kustomization.yaml new file mode 100644 index 00000000..12f27557 --- /dev/null +++ b/src/web/deploy/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default + +resources: + - ui-deployment.yaml + - ui-service.yaml \ No newline at end of file diff --git a/src/web/deploy/ui-deployment.yaml b/src/web/deploy/ui-deployment.yaml new file mode 100644 index 00000000..1004678c --- /dev/null +++ b/src/web/deploy/ui-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ui +spec: + replicas: 1 + selector: + matchLabels: + app: ui + template: + metadata: + labels: + app: ui + spec: + containers: + - name: ui + image: cognixacr.azurecr.io/cognix/ui:main + imagePullPolicy: Always + env: + - name: VITE_PLATFORM_API_URL + value: https://rag.cognix.ch + envFrom: + - configMapRef: + name: web-srv + ports: + - containerPort: 80 + imagePullSecrets: + - name: pullsecret \ No newline at end of file diff --git a/src/web/deploy/ui-service.yaml b/src/web/deploy/ui-service.yaml new file mode 100644 index 00000000..6f84d765 --- /dev/null +++ b/src/web/deploy/ui-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: ui +spec: + selector: + app: ui + ports: + - protocol: TCP + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/src/web/docker-compose-cloud.yaml b/src/web/docker-compose-cloud.yaml new file mode 100644 index 00000000..ffe3a9d5 --- /dev/null +++ b/src/web/docker-compose-cloud.yaml @@ -0,0 +1,7 @@ +services: + ui: + image: cognix:ui + build: + context: . + dockerfile: Dockerfile.dev + network_mode: host diff --git a/src/web/docker-compose.yaml b/src/web/docker-compose.yaml new file mode 100644 index 00000000..caaf506e --- /dev/null +++ b/src/web/docker-compose.yaml @@ -0,0 +1,14 @@ +services: + ui: + image: cognix:ui + build: + context: . + dockerfile: Dockerfile + ports: + - "80:80" + networks: + - dev-network + +networks: + dev-network: + external: true \ No newline at end of file diff --git a/index.html b/src/web/index.html similarity index 71% rename from index.html rename to src/web/index.html index e4b78eae..ff8261e8 100644 --- a/index.html +++ b/src/web/index.html @@ -2,9 +2,9 @@ - + - Vite + React + TS + CogniX
diff --git a/src/web/nginx/nginx-cloud.conf b/src/web/nginx/nginx-cloud.conf new file mode 100644 index 00000000..9ecfa3ba --- /dev/null +++ b/src/web/nginx/nginx-cloud.conf @@ -0,0 +1,19 @@ +worker_processes 1; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + server { + listen 80; + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html?$args; + index index.html index.htm; + } + location /api { + proxy_pass https://rag.cognix.ch/api; + } + + } +} diff --git a/src/web/nginx/nginx-dev.conf b/src/web/nginx/nginx-dev.conf new file mode 100644 index 00000000..634f1af6 --- /dev/null +++ b/src/web/nginx/nginx-dev.conf @@ -0,0 +1,19 @@ +worker_processes 1; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + server { + listen 80; + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html?$args; + index index.html index.htm; + } + location /api { + proxy_pass http://api:8080/api; + } + + } +} diff --git a/src/web/nginx/nginx.conf b/src/web/nginx/nginx.conf new file mode 100644 index 00000000..92e711ed --- /dev/null +++ b/src/web/nginx/nginx.conf @@ -0,0 +1,15 @@ +worker_processes 1; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + server { + listen 80; + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html?$args; + index index.html index.htm; + } + } +} diff --git a/package-lock.json b/src/web/package-lock.json similarity index 50% rename from package-lock.json rename to src/web/package-lock.json index da242eb3..0bfba5b7 100644 --- a/package-lock.json +++ b/src/web/package-lock.json @@ -8,23 +8,44 @@ "name": "cognix", "version": "0.0.0", "dependencies": { + "@hookform/resolvers": "^3.3.4", + "@react-oauth/google": "^0.12.1", + "@tanstack/react-table": "^8.16.0", + "autoprefixer": "^10.4.19", + "axios": "^1.6.8", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.0", + "date-fns": "^3.6.0", + "dotenv": "^16.4.5", + "lucide-react": "^0.367.0", "path": "^0.12.7", + "postcss": "^8.4.38", + "radix-ui": "^1.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "react-router-dom": "^6.22.3" + "react-hook-form": "^7.51.3", + "react-router-dom": "^6.22.3", + "react-toastify": "^10.0.5", + "swr": "^2.2.5", + "tailwind-merge": "^2.2.2", + "uuid": "^9.0.1", + "zod": "^3.23.4" }, "devDependencies": { "@types/node": "^20.12.6", "@types/react": "^18.2.66", "@types/react-dom": "^18.2.22", + "@types/uuid": "^9.0.8", "@typescript-eslint/eslint-plugin": "^7.2.0", "@typescript-eslint/parser": "^7.2.0", "@vitejs/plugin-react": "^4.2.1", "eslint": "^8.57.0", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-react-refresh": "^0.4.6", + "tailwindcss": "^3.4.3", "typescript": "^5.2.2", - "vite": "^5.2.0" + "vite": "^5.2.0", + "vite-plugin-svgr": "^4.2.0" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -36,6 +57,18 @@ "node": ">=0.10.0" } }, + "node_modules/@alloc/quick-lru": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", + "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -346,6 +379,17 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.4.tgz", + "integrity": "sha512-dkxf7+hn8mFBwKjs9bvBlArzLVxVbS8usaPUDd5p2a9JCL9tB8OaOVN1isD4+Xyk4ns89/xeOmbQvgdK7IIVdA==", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.24.0", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz", @@ -856,6 +900,48 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@floating-ui/core": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.0.tgz", + "integrity": "sha512-PcF++MykgmTj3CIyOQbKA/hDzOAiqI3mhuoN44WRCopIs1sgoDoU4oty4Jtqaj/y3oDU6fnVSm4QG0a3t5i0+g==", + "dependencies": { + "@floating-ui/utils": "^0.2.1" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.3.tgz", + "integrity": "sha512-RnDthu3mzPlQ31Ss/BTwQ1zjzIhr3lk1gZB1OC56h/1vEtaXkESrOqL5fQVMfXpwGtRwX+YsZBdyHtJMQnkArw==", + "dependencies": { + "@floating-ui/core": "^1.0.0", + "@floating-ui/utils": "^0.2.0" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.0.8.tgz", + "integrity": "sha512-HOdqOt3R3OGeTKidaLvJKcgg75S6tibQ3Tif4eyd91QnIJWr0NLvoXFpJA/j8HqkFSL68GDca9AuyWEHlhyClw==", + "dependencies": { + "@floating-ui/dom": "^1.6.1" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.1.tgz", + "integrity": "sha512-9TANp6GPoMtYzQdt54kfAyMmz1+osLlXdg2ENroU7zzrtflTLrrC/lgrIfaSe+Wu0b89GKccT7vxXA0MoAIO+Q==" + }, + "node_modules/@hookform/resolvers": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-3.3.4.tgz", + "integrity": "sha512-o5cgpGOuJYrd+iMKvkttOclgwRW86EsWJZZRC23prf0uU2i48Htq4PuT73AVb9ionFyZrwYEITuOFGF+BydEtQ==", + "peerDependencies": { + "react-hook-form": "^7.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -911,6 +997,50 @@ "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", "dev": true }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -994,138 +1124,1534 @@ "node": ">= 8" } }, - "node_modules/@remix-run/router": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.15.3.tgz", - "integrity": "sha512-Oy8rmScVrVxWZVOpEF57ovlnhpZ8CCPlnIIumVcV9nFdiSIrus99+Lw78ekXyGvVDlIsFJbSfmSovJUhCWYV3w==", + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "optional": true, "engines": { - "node": ">=14.0.0" + "node": ">=14" } }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.14.1.tgz", - "integrity": "sha512-fH8/o8nSUek8ceQnT7K4EQbSiV7jgkHq81m9lWZFIXjJ7lJzpWXbQFpT/Zh6OZYnpFykvzC3fbEvEAFZu03dPA==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ] + "node_modules/@radix-ui/number": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.0.1.tgz", + "integrity": "sha512-T5gIdVO2mmPW3NNhjNgEP3cqMXjXL9UbO0BzWcXfvdBs+BohbQxvd/K5hSVKmn9/lbTdsQVKbUcP5WLCwvUbBg==", + "dependencies": { + "@babel/runtime": "^7.13.10" + } }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.14.1.tgz", - "integrity": "sha512-Y/9OHLjzkunF+KGEoJr3heiD5X9OLa8sbT1lm0NYeKyaM3oMhhQFvPB0bNZYJwlq93j8Z6wSxh9+cyKQaxS7PQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ] + "node_modules/@radix-ui/primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.0.1.tgz", + "integrity": "sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==", + "dependencies": { + "@babel/runtime": "^7.13.10" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.14.1.tgz", - "integrity": "sha512-+kecg3FY84WadgcuSVm6llrABOdQAEbNdnpi5X3UwWiFVhZIZvKgGrF7kmLguvxHNQy+UuRV66cLVl3S+Rkt+Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ] + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.0.3.tgz", + "integrity": "sha512-duVGKeWPSUILr/MdlPxV+GeULTc2rS1aihGdQ3N2qCUPMgxYLxvAsHJM3mCVLF8d5eK+ympmB22mb1F3a5biNw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-visually-hidden": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.14.1.tgz", - "integrity": "sha512-2pYRzEjVqq2TB/UNv47BV/8vQiXkFGVmPFwJb+1E0IFFZbIX8/jo1olxqqMbo6xCXf8kabANhp5bzCij2tFLUA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ] + "node_modules/@radix-ui/react-accordion": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.1.2.tgz", + "integrity": "sha512-fDG7jcoNKVjSK6yfmuAs0EnPDro0WMXIhMtXdTBWqEioVW206ku+4Lw07e+13lUkFkpoEQ2PdeMIAGpdqEAmDg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collapsible": "1.0.3", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.14.1.tgz", - "integrity": "sha512-mS6wQ6Do6/wmrF9aTFVpIJ3/IDXhg1EZcQFYHZLHqw6AzMBjTHWnCG35HxSqUNphh0EHqSM6wRTT8HsL1C0x5g==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ] + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.0.5.tgz", + "integrity": "sha512-OrVIOcZL0tl6xibeuGt5/+UxoT2N27KCFOPjFyfXMnchxSHZ/OW7cCX2nGlIYJrbHK/fczPcFzAwvNBB6XBNMA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-dialog": "1.0.5", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-slot": "1.0.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.14.1.tgz", - "integrity": "sha512-p9rGKYkHdFMzhckOTFubfxgyIO1vw//7IIjBBRVzyZebWlzRLeNhqxuSaZ7kCEKVkm/kuC9fVRW9HkC/zNRG2w==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ] + "node_modules/@radix-ui/react-arrow": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.0.3.tgz", + "integrity": "sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.14.1.tgz", - "integrity": "sha512-nDY6Yz5xS/Y4M2i9JLQd3Rofh5OR8Bn8qe3Mv/qCVpHFlwtZSBYSPaU4mrGazWkXrdQ98GB//H0BirGR/SKFSw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ] + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.0.3.tgz", + "integrity": "sha512-fXR5kbMan9oQqMuacfzlGG/SQMcmMlZ4wrvpckv8SgUulD0MMpspxJrxg/Gp/ISV3JfV1AeSWTYK9GvxA4ySwA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.14.1.tgz", - "integrity": "sha512-im7HE4VBL+aDswvcmfx88Mp1soqL9OBsdDBU8NqDEYtkri0qV0THhQsvZtZeNNlLeCUQ16PZyv7cqutjDF35qw==", - "cpu": [ - "ppc64le" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ] + "node_modules/@radix-ui/react-avatar": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.0.4.tgz", + "integrity": "sha512-kVK2K7ZD3wwj3qhle0ElXhOjbezIgyl2hVvgwfIdexL3rN6zJmy5AqqIf+D31lxVppdzV8CjAfZ6PklkmInZLw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.14.1.tgz", - "integrity": "sha512-RWdiHuAxWmzPJgaHJdpvUUlDz8sdQz4P2uv367T2JocdDa98iRw2UjIJ4QxSyt077mXZT2X6pKfT2iYtVEvOFw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ] + "node_modules/@radix-ui/react-checkbox": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.0.4.tgz", + "integrity": "sha512-CBuGQa52aAYnADZVt/KBQzXrwx6TqnlwtcIPGtVt5JkkzQwMOLJjPukimhfKEr4GQNd43C+djUh5Ikopj8pSLg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-previous": "1.0.1", + "@radix-ui/react-use-size": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.14.1.tgz", - "integrity": "sha512-VMgaGQ5zRX6ZqV/fas65/sUGc9cPmsntq2FiGmayW9KMNfWVG/j0BAqImvU4KTeOOgYSf1F+k6at1UfNONuNjA==", - "cpu": [ - "s390x" - ], + "node_modules/@radix-ui/react-collapsible": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.0.3.tgz", + "integrity": "sha512-UBmVDkmR6IvDsloHVN+3rtx4Mi5TFvylYXpluuv0f37dtaz3H99bp8No0LGXRigVpl3UAT4l9j6bIchh42S/Gg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.0.3.tgz", + "integrity": "sha512-3SzW+0PW7yBBoQlT8wNcGtaxaD0XSu0uLUFgrtHY08Acx05TaHaOmVLR73c0j/cqpDy53KBMO7s0dx2wmOIDIA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-slot": "1.0.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz", + "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz", + "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.1.5.tgz", + "integrity": "sha512-R5XaDj06Xul1KGb+WP8qiOh7tKJNz2durpLBXAGZjSVtctcRFCuEvy2gtMwRJGePwQQE5nV77gs4FwRi8T+r2g==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-menu": "2.0.6", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.0.5.tgz", + "integrity": "sha512-GjWJX/AUpB703eEBanuBnIWdIXg6NvJFCXcNlSZk4xdszCdhrJgBoUd1cGk67vFO+WdA2pfI/plOpqz/5GUP6Q==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-focus-guards": "1.0.1", + "@radix-ui/react-focus-scope": "1.0.4", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-slot": "1.0.2", + "@radix-ui/react-use-controllable-state": "1.0.1", + "aria-hidden": "^1.1.1", + "react-remove-scroll": "2.5.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.0.1.tgz", + "integrity": "sha512-RXcvnXgyvYvBEOhCBuddKecVkoMiI10Jcm5cTI7abJRAHYfFxeu+FBQs/DvdxSYucxR5mna0dNsL6QFlds5TMA==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz", + "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-escape-keydown": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.0.6.tgz", + "integrity": "sha512-i6TuFOoWmLWq+M/eCLGd/bQ2HfAX1RJgvrBQ6AQLmzfvsLdefxbWu8G9zczcPFfcSPehz9GcpF6K9QYreFV8hA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-menu": "2.0.6", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz", + "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz", + "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.0.7.tgz", + "integrity": "sha512-OcUN2FU0YpmajD/qkph3XzMcK/NmSk9hGWnjV68p6QiZMgILugusgQwnLSDs3oFSJYGKf3Y49zgFedhGh04k9A==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-popper": "1.1.3", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz", + "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.0.2.tgz", + "integrity": "sha512-N5ehvlM7qoTLx7nWPodsPYPgMzA5WM8zZChQg8nyFJKnDO5WHdba1vv5/H6IO5LtJMfD2Q3wh1qHFGNtK0w3bQ==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.0.6.tgz", + "integrity": "sha512-BVkFLS+bUC8HcImkRKPSiVumA1VPOOEC5WBMiT+QAVsPzW1FJzI9KnqgGxVDPBcql5xXrHkD3JOVoXWEXD8SYA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-focus-guards": "1.0.1", + "@radix-ui/react-focus-scope": "1.0.4", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-popper": "1.1.3", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-roving-focus": "1.0.4", + "@radix-ui/react-slot": "1.0.2", + "@radix-ui/react-use-callback-ref": "1.0.1", + "aria-hidden": "^1.1.1", + "react-remove-scroll": "2.5.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.1.4.tgz", + "integrity": "sha512-Cc+seCS3PmWmjI51ufGG7zp1cAAIRqHVw7C9LOA2TZ+R4hG6rDvHcTqIsEEFLmZO3zNVH72jOOE7kKNy8W+RtA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1", + "@radix-ui/react-use-previous": "1.0.1", + "@radix-ui/react-visually-hidden": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.0.7.tgz", + "integrity": "sha512-shtvVnlsxT6faMnK/a7n0wptwBD23xc1Z5mdrtKLwVEfsEMXodS0r5s0/g5P0hX//EKYZS2sxUjqfzlg52ZSnQ==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-focus-guards": "1.0.1", + "@radix-ui/react-focus-scope": "1.0.4", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-popper": "1.1.3", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-slot": "1.0.2", + "@radix-ui/react-use-controllable-state": "1.0.1", + "aria-hidden": "^1.1.1", + "react-remove-scroll": "2.5.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz", + "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1", + "@radix-ui/react-use-rect": "1.0.1", + "@radix-ui/react-use-size": "1.0.1", + "@radix-ui/rect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz", + "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz", + "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz", + "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-slot": "1.0.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.0.3.tgz", + "integrity": "sha512-5G6Om/tYSxjSeEdrb1VfKkfZfn/1IlPWd731h2RfPuSbIfNUgfqAwbKfJCg/PP6nuUCTrYzalwHSpSinoWoCag==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.1.3.tgz", + "integrity": "sha512-x+yELayyefNeKeTx4fjK6j99Fs6c4qKm3aY38G3swQVTN6xMpsrbigC0uHs2L//g8q4qR7qOcww8430jJmi2ag==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-roving-focus": "1.0.4", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-previous": "1.0.1", + "@radix-ui/react-use-size": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.0.4.tgz", + "integrity": "sha512-2mUg5Mgcu001VkGy+FfzZyzbmuUWzgWkj3rvv4yu+mLw03+mTzbxZHvfcGyFp2b8EkQeMkpRQ5FiA2Vr2O6TeQ==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.0.5.tgz", + "integrity": "sha512-b6PAgH4GQf9QEn8zbT2XUHpW5z8BzqEc7Kl11TwDrvuTrxlkcjTD5qa/bxgKr+nmuXKu4L/W5UZ4mlP/VG/5Gw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/number": "1.0.1", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.0.0.tgz", + "integrity": "sha512-RH5b7af4oHtkcHS7pG6Sgv5rk5Wxa7XI8W5gvB1N/yiuDGZxko1ynvOiVhFM7Cis2A8zxF9bTOUVbRDzPepe6w==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/number": "1.0.1", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-focus-guards": "1.0.1", + "@radix-ui/react-focus-scope": "1.0.4", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-popper": "1.1.3", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-slot": "1.0.2", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1", + "@radix-ui/react-use-previous": "1.0.1", + "@radix-ui/react-visually-hidden": "1.0.3", + "aria-hidden": "^1.1.1", + "react-remove-scroll": "2.5.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.0.3.tgz", + "integrity": "sha512-itYmTy/kokS21aiV5+Z56MZB54KrhPgn6eHDKkFeOLR34HMN2s8PaN47qZZAGnvupcjxHaFZnW4pQEh0BvvVuw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.1.2.tgz", + "integrity": "sha512-NKs15MJylfzVsCagVSWKhGGLNR1W9qWs+HtgbmjjVUB3B9+lb3PYoXxVju3kOrpf0VKyVCtZp+iTwVoqpa1Chw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/number": "1.0.1", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1", + "@radix-ui/react-use-previous": "1.0.1", + "@radix-ui/react-use-size": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz", + "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-compose-refs": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.0.3.tgz", + "integrity": "sha512-mxm87F88HyHztsI7N+ZUmEoARGkC22YVW5CaC+Byc+HRpuvCrOBPTAnXgf+tZ/7i0Sg/eOePGdMhUKhPaQEqow==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-previous": "1.0.1", + "@radix-ui/react-use-size": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.0.4.tgz", + "integrity": "sha512-egZfYY/+wRNCflXNHx+dePvnz9FbmssDTJBtgRfDY7e8SE5oIo3Py2eCB1ckAbh1Q7cQ/6yJZThJ++sgbxibog==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-roving-focus": "1.0.4", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.1.5.tgz", + "integrity": "sha512-fRLn227WHIBRSzuRzGJ8W+5YALxofH23y0MlPLddaIpLpCDqdE0NZlS2NRQDRiptfxDeeCjgFIpexB1/zkxDlw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-collection": "1.0.3", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1", + "@radix-ui/react-visually-hidden": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.0.3.tgz", + "integrity": "sha512-Pkqg3+Bc98ftZGsl60CLANXQBBQ4W3mTFS9EJvNxKMZ7magklKV69/id1mlAlOFDDfHvlCms0fx8fA4CMKDJHg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.0.4.tgz", + "integrity": "sha512-Uaj/M/cMyiyT9Bx6fOZO0SAG4Cls0GptBWiBmBxofmDbNVnYYoyRWj/2M/6VCi/7qcXFWnHhRUfdfZFvvkuu8A==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-roving-focus": "1.0.4", + "@radix-ui/react-toggle": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.0.4.tgz", + "integrity": "sha512-tBgmM/O7a07xbaEkYJWYTXkIdU/1pW4/KZORR43toC/4XWyBCURK0ei9kMUdp+gTPPKBgYLxXmRSH1EVcIDp8Q==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-roving-focus": "1.0.4", + "@radix-ui/react-separator": "1.0.3", + "@radix-ui/react-toggle-group": "1.0.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz", + "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-dismissable-layer": "1.0.5", + "@radix-ui/react-id": "1.0.1", + "@radix-ui/react-popper": "1.1.3", + "@radix-ui/react-portal": "1.0.4", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-slot": "1.0.2", + "@radix-ui/react-use-controllable-state": "1.0.1", + "@radix-ui/react-visually-hidden": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz", + "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz", + "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-use-callback-ref": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz", + "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-use-callback-ref": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz", + "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.0.1.tgz", + "integrity": "sha512-cV5La9DPwiQ7S0gf/0qiD6YgNqM5Fk97Kdrlc5yBcrF3jyEZQwm7vYFqMo4IfeHgJXsRaMvLABFtd0OVEmZhDw==", + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz", + "integrity": "sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/rect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.0.1.tgz", + "integrity": "sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.0.3.tgz", + "integrity": "sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-primitive": "1.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.0.1.tgz", + "integrity": "sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==", + "dependencies": { + "@babel/runtime": "^7.13.10" + } + }, + "node_modules/@react-oauth/google": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@react-oauth/google/-/google-0.12.1.tgz", + "integrity": "sha512-qagsy22t+7UdkYAiT5ZhfM4StXi9PPNvw0zuwNmabrWyMKddczMtBIOARflbaIj+wHiQjnMAsZmzsUYuXeyoSg==", + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@remix-run/router": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.15.3.tgz", + "integrity": "sha512-Oy8rmScVrVxWZVOpEF57ovlnhpZ8CCPlnIIumVcV9nFdiSIrus99+Lw78ekXyGvVDlIsFJbSfmSovJUhCWYV3w==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@rollup/pluginutils": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.0.tgz", + "integrity": "sha512-XTIWOPPcpvyKI6L1NHo0lFlCyznUEyPmPY1mc3KpPVDYulHSTvyeLNVW00QTLIAFNhR3kYnJTQHeGqU4M3n09g==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.14.1.tgz", + "integrity": "sha512-fH8/o8nSUek8ceQnT7K4EQbSiV7jgkHq81m9lWZFIXjJ7lJzpWXbQFpT/Zh6OZYnpFykvzC3fbEvEAFZu03dPA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.14.1.tgz", + "integrity": "sha512-Y/9OHLjzkunF+KGEoJr3heiD5X9OLa8sbT1lm0NYeKyaM3oMhhQFvPB0bNZYJwlq93j8Z6wSxh9+cyKQaxS7PQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.14.1.tgz", + "integrity": "sha512-+kecg3FY84WadgcuSVm6llrABOdQAEbNdnpi5X3UwWiFVhZIZvKgGrF7kmLguvxHNQy+UuRV66cLVl3S+Rkt+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.14.1.tgz", + "integrity": "sha512-2pYRzEjVqq2TB/UNv47BV/8vQiXkFGVmPFwJb+1E0IFFZbIX8/jo1olxqqMbo6xCXf8kabANhp5bzCij2tFLUA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.14.1.tgz", + "integrity": "sha512-mS6wQ6Do6/wmrF9aTFVpIJ3/IDXhg1EZcQFYHZLHqw6AzMBjTHWnCG35HxSqUNphh0EHqSM6wRTT8HsL1C0x5g==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.14.1.tgz", + "integrity": "sha512-p9rGKYkHdFMzhckOTFubfxgyIO1vw//7IIjBBRVzyZebWlzRLeNhqxuSaZ7kCEKVkm/kuC9fVRW9HkC/zNRG2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.14.1.tgz", + "integrity": "sha512-nDY6Yz5xS/Y4M2i9JLQd3Rofh5OR8Bn8qe3Mv/qCVpHFlwtZSBYSPaU4mrGazWkXrdQ98GB//H0BirGR/SKFSw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.14.1.tgz", + "integrity": "sha512-im7HE4VBL+aDswvcmfx88Mp1soqL9OBsdDBU8NqDEYtkri0qV0THhQsvZtZeNNlLeCUQ16PZyv7cqutjDF35qw==", + "cpu": [ + "ppc64le" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.14.1.tgz", + "integrity": "sha512-RWdiHuAxWmzPJgaHJdpvUUlDz8sdQz4P2uv367T2JocdDa98iRw2UjIJ4QxSyt077mXZT2X6pKfT2iYtVEvOFw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.14.1.tgz", + "integrity": "sha512-VMgaGQ5zRX6ZqV/fas65/sUGc9cPmsntq2FiGmayW9KMNfWVG/j0BAqImvU4KTeOOgYSf1F+k6at1UfNONuNjA==", + "cpu": [ + "s390x" + ], "dev": true, "optional": true, "os": [ @@ -1197,6 +2723,250 @@ "win32" ] }, + "node_modules/@svgr/babel-plugin-add-jsx-attribute": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-8.0.0.tgz", + "integrity": "sha512-b9MIk7yhdS1pMCZM8VeNfUlSKVRhsHZNMl5O9SfaX0l0t5wjdgu4IDzGB8bpnGBBOjGST3rRFVsaaEtI4W6f7g==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-remove-jsx-attribute": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-8.0.0.tgz", + "integrity": "sha512-BcCkm/STipKvbCl6b7QFrMh/vx00vIP63k2eM66MfHJzPr6O2U0jYEViXkHJWqXqQYjdeA9cuCl5KWmlwjDvbA==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-remove-jsx-empty-expression": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-8.0.0.tgz", + "integrity": "sha512-5BcGCBfBxB5+XSDSWnhTThfI9jcO5f0Ai2V24gZpG+wXF14BzwxxdDb4g6trdOux0rhibGs385BeFMSmxtS3uA==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-replace-jsx-attribute-value": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-8.0.0.tgz", + "integrity": "sha512-KVQ+PtIjb1BuYT3ht8M5KbzWBhdAjjUPdlMtpuw/VjT8coTrItWX6Qafl9+ji831JaJcu6PJNKCV0bp01lBNzQ==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-svg-dynamic-title": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-8.0.0.tgz", + "integrity": "sha512-omNiKqwjNmOQJ2v6ge4SErBbkooV2aAWwaPFs2vUY7p7GhVkzRkJ00kILXQvRhA6miHnNpXv7MRnnSjdRjK8og==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-svg-em-dimensions": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-8.0.0.tgz", + "integrity": "sha512-mURHYnu6Iw3UBTbhGwE/vsngtCIbHE43xCRK7kCw4t01xyGqb2Pd+WXekRRoFOBIY29ZoOhUCTEweDMdrjfi9g==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-transform-react-native-svg": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-8.1.0.tgz", + "integrity": "sha512-Tx8T58CHo+7nwJ+EhUwx3LfdNSG9R2OKfaIXXs5soiy5HtgoAEkDay9LIimLOcG8dJQH1wPZp/cnAv6S9CrR1Q==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-transform-svg-component": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-8.0.0.tgz", + "integrity": "sha512-DFx8xa3cZXTdb/k3kfPeaixecQLgKh5NVBMwD0AQxOzcZawK4oo1Jh9LbrcACUivsCA7TLG8eeWgrDXjTMhRmw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-preset": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-preset/-/babel-preset-8.1.0.tgz", + "integrity": "sha512-7EYDbHE7MxHpv4sxvnVPngw5fuR6pw79SkcrILHJ/iMpuKySNCl5W1qcwPEpU+LgyRXOaAFgH0KhwD18wwg6ug==", + "dev": true, + "dependencies": { + "@svgr/babel-plugin-add-jsx-attribute": "8.0.0", + "@svgr/babel-plugin-remove-jsx-attribute": "8.0.0", + "@svgr/babel-plugin-remove-jsx-empty-expression": "8.0.0", + "@svgr/babel-plugin-replace-jsx-attribute-value": "8.0.0", + "@svgr/babel-plugin-svg-dynamic-title": "8.0.0", + "@svgr/babel-plugin-svg-em-dimensions": "8.0.0", + "@svgr/babel-plugin-transform-react-native-svg": "8.1.0", + "@svgr/babel-plugin-transform-svg-component": "8.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/core": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz", + "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==", + "dev": true, + "dependencies": { + "@babel/core": "^7.21.3", + "@svgr/babel-preset": "8.1.0", + "camelcase": "^6.2.0", + "cosmiconfig": "^8.1.3", + "snake-case": "^3.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + } + }, + "node_modules/@svgr/hast-util-to-babel-ast": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-8.0.0.tgz", + "integrity": "sha512-EbDKwO9GpfWP4jN9sGdYwPBU0kdomaPIL2Eu4YwmgP+sJeXT+L7bMwJUBnhzfH8Q2qMBqZ4fJwpCyYsAN3mt2Q==", + "dev": true, + "dependencies": { + "@babel/types": "^7.21.3", + "entities": "^4.4.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + } + }, + "node_modules/@svgr/plugin-jsx": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/plugin-jsx/-/plugin-jsx-8.1.0.tgz", + "integrity": "sha512-0xiIyBsLlr8quN+WyuxooNW9RJ0Dpr8uOnH/xrCVO8GLUcwHISwj1AG0k+LFzteTkAA0GbX0kj9q6Dk70PTiPA==", + "dev": true, + "dependencies": { + "@babel/core": "^7.21.3", + "@svgr/babel-preset": "8.1.0", + "@svgr/hast-util-to-babel-ast": "8.0.0", + "svg-parser": "^2.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@svgr/core": "*" + } + }, + "node_modules/@tanstack/react-table": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.16.0.tgz", + "integrity": "sha512-rKRjnt8ostqN2fercRVOIH/dq7MAmOENCMvVlKx6P9Iokhh6woBGnIZEkqsY/vEJf1jN3TqLOb34xQGLVRuhAg==", + "dependencies": { + "@tanstack/table-core": "8.16.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, + "node_modules/@tanstack/table-core": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.16.0.tgz", + "integrity": "sha512-dCG8vQGk4js5v88/k83tTedWOwjGnIyONrKpHpfmSJB8jwFHl8GSu1sBBxbtACVAPtAQgwNxl0rw1d3RqRM1Tg==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -1263,13 +3033,13 @@ "version": "15.7.12", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz", "integrity": "sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==", - "dev": true + "devOptional": true }, "node_modules/@types/react": { "version": "18.2.74", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.74.tgz", "integrity": "sha512-9AEqNZZyBx8OdZpxzQlaFEVCSFUM2YXJH46yPOiOpm078k6ZLOCcuAzGum/zK8YBwY+dbahVNbHrbgrAwIRlqw==", - "dev": true, + "devOptional": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.0.2" @@ -1279,7 +3049,7 @@ "version": "18.2.24", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.24.tgz", "integrity": "sha512-cN6upcKd8zkGy4HU9F1+/s98Hrp6D4MOcippK4PoE8OZRngohHZpbJn1GsaDLz87MqvHNoT13nHvNqM9ocRHZg==", - "dev": true, + "devOptional": true, "dependencies": { "@types/react": "*" } @@ -1290,6 +3060,12 @@ "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", "dev": true }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "dev": true + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.5.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.5.0.tgz", @@ -1563,12 +3339,48 @@ "node": ">=4" } }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "node_modules/aria-hidden": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.4.tgz", + "integrity": "sha512-y+CcFFwelSXpLZk/7fMB2mUbGtX9lKycf1MWJ7CaTIERyitVlyQx6C+sxcROU2BAJ24OiZyK+8wj2i8AlBoS3A==", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -1578,12 +3390,75 @@ "node": ">=8" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/autoprefixer": { + "version": "10.4.19", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.19.tgz", + "integrity": "sha512-BaENR2+zBZ8xXhM4pUaKUxlVdxZ0EZhjvbopwnXmxRUfqDmwSpC2lAi/QXvx7NRdPCo1WKEcEF6mV64si1z4Ew==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "browserslist": "^4.23.0", + "caniuse-lite": "^1.0.30001599", + "fraction.js": "^4.3.7", + "normalize-range": "^0.1.2", + "picocolors": "^1.0.0", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/axios": { + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", + "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -1609,7 +3484,6 @@ "version": "4.23.0", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", - "dev": true, "funding": [ { "type": "opencollective", @@ -1646,11 +3520,31 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/camelcase-css": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", + "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001607", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001607.tgz", "integrity": "sha512-WcvhVRjXLKFB/kmOFVwELtMxyhq3iM/MvmXcyCe2PNf166c39mptscOc/45TTS96n2gpNV2z7+NakArTWZCQ3w==", - "dev": true, "funding": [ { "type": "opencollective", @@ -1672,12 +3566,80 @@ "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", "dev": true, "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" }, "engines": { - "node": ">=4" + "node": ">= 6" + } + }, + "node_modules/class-variance-authority": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.0.tgz", + "integrity": "sha512-jFI8IQw4hczaL4ALINxqLEXQbWcNjoSkloa4IaufXCJr6QawJyw7tuRysRsrE8w2p/4gGaxKIt/hX3qz/IbD1A==", + "dependencies": { + "clsx": "2.0.0" + }, + "funding": { + "url": "https://joebell.co.uk" + } + }, + "node_modules/class-variance-authority/node_modules/clsx": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.0.0.tgz", + "integrity": "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q==", + "engines": { + "node": ">=6" + } + }, + "node_modules/client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" + }, + "node_modules/clsx": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz", + "integrity": "sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==", + "engines": { + "node": ">=6" } }, "node_modules/color-convert": { @@ -1695,6 +3657,26 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1707,6 +3689,32 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "dev": true, + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -1721,11 +3729,32 @@ "node": ">= 8" } }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", - "dev": true + "devOptional": true + }, + "node_modules/date-fns": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", + "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" + } }, "node_modules/debug": { "version": "4.3.4", @@ -1750,6 +3779,25 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==" + }, + "node_modules/didyoumean": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", + "dev": true + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -1762,6 +3810,12 @@ "node": ">=8" } }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -1774,12 +3828,65 @@ "node": ">=6.0.0" } }, + "node_modules/dot-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", + "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", + "dev": true, + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/electron-to-chromium": { "version": "1.4.729", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.729.tgz", - "integrity": "sha512-bx7+5Saea/qu14kmPTDHQxkp2UnziG3iajUQu3BxFvCOnpAJdDbMV4rSl+EqFDkkpNNVUFlR1kDfpL59xfy1HA==", + "integrity": "sha512-bx7+5Saea/qu14kmPTDHQxkp2UnziG3iajUQu3BxFvCOnpAJdDbMV4rSl+EqFDkkpNNVUFlR1kDfpL59xfy1HA==" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/esbuild": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", @@ -1822,7 +3929,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", - "dev": true, "engines": { "node": ">=6" } @@ -2109,6 +4215,12 @@ "node": ">=4.0" } }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -2233,6 +4345,66 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fraction.js": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", + "integrity": "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==", + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://github.com/sponsors/rawify" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2253,6 +4425,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -2262,6 +4443,14 @@ "node": ">=6.9.0" } }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "engines": { + "node": ">=6" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -2360,6 +4549,18 @@ "node": ">=4" } }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", @@ -2410,6 +4611,44 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "node_modules/invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dependencies": { + "loose-envify": "^1.0.0" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "dev": true, + "dependencies": { + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -2419,6 +4658,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -2455,6 +4703,33 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jiti": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz", + "integrity": "sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==", + "dev": true, + "bin": { + "jiti": "bin/jiti.js" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2490,6 +4765,12 @@ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -2536,6 +4817,21 @@ "node": ">= 0.8.0" } }, + "node_modules/lilconfig": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", + "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -2568,6 +4864,15 @@ "loose-envify": "cli.js" } }, + "node_modules/lower-case": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", + "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", + "dev": true, + "dependencies": { + "tslib": "^2.0.3" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -2577,6 +4882,14 @@ "yallist": "^3.0.2" } }, + "node_modules/lucide-react": { + "version": "0.367.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.367.0.tgz", + "integrity": "sha512-3FWiBaJiqMrx5a1sjH3CVdPqWnw/Z/PTVeeTDmOeILSs+8Ah+VhCd4FQMeHo6Z0WxHcm9piIOtilQwvceiCCKQ==", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -2599,6 +4912,25 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", @@ -2614,17 +4946,36 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, "node_modules/nanoid": { "version": "3.3.7", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", - "dev": true, "funding": [ { "type": "github", @@ -2644,11 +4995,55 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/no-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", + "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", + "dev": true, + "dependencies": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + } + }, "node_modules/node-releases": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", - "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", - "dev": true + "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "dev": true, + "engines": { + "node": ">= 6" + } }, "node_modules/once": { "version": "1.4.0", @@ -2718,6 +5113,24 @@ "node": ">=6" } }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/path": { "version": "0.12.7", "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", @@ -2754,6 +5167,37 @@ "node": ">=8" } }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dev": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "dev": true, + "engines": { + "node": "14 || >=16.14" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -2766,8 +5210,7 @@ "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", - "dev": true + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -2781,11 +5224,28 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pirates": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", + "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/postcss": { "version": "8.4.38", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", - "dev": true, "funding": [ { "type": "opencollective", @@ -2809,6 +5269,126 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postcss-import": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz", + "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.0.0", + "read-cache": "^1.0.0", + "resolve": "^1.1.7" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-js": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz", + "integrity": "sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==", + "dev": true, + "dependencies": { + "camelcase-css": "^2.0.1" + }, + "engines": { + "node": "^12 || ^14 || >= 16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.4.21" + } + }, + "node_modules/postcss-load-config": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.2.tgz", + "integrity": "sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "lilconfig": "^3.0.0", + "yaml": "^2.3.4" + }, + "engines": { + "node": ">= 14" + }, + "peerDependencies": { + "postcss": ">=8.0.9", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "postcss": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/postcss-load-config/node_modules/lilconfig": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.1.tgz", + "integrity": "sha512-O18pf7nyvHTckunPWCV1XUNXU1piu01y2b7ATJ0ppkUkk8ocqVWBrYjJBCwHDjD/ZWcfyrA0P4gKhzWGi5EINQ==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/postcss-nested": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.1.tgz", + "integrity": "sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.11" + }, + "engines": { + "node": ">=12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.16", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.16.tgz", + "integrity": "sha512-A0RVJrX+IUkVZbW3ClroRWurercFhieevHB38sr2+l9eUClMqome3LmEmnhlNy+5Mr2EYN6B2Kaw9wYdd+VHiw==", + "dev": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==" + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2826,6 +5406,11 @@ "node": ">= 0.6.0" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -2855,6 +5440,48 @@ } ] }, + "node_modules/radix-ui": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.0.1.tgz", + "integrity": "sha512-qfGibbqtbOlxP3b+1JjbLUc8Q7+e9DL8gFycLtkBkoAQyUkKuHAEBfFUcyG5MaQHjqRuML+YLtt/R1/dUYQafQ==", + "dependencies": { + "@radix-ui/react-accessible-icon": "latest", + "@radix-ui/react-accordion": "latest", + "@radix-ui/react-alert-dialog": "latest", + "@radix-ui/react-aspect-ratio": "latest", + "@radix-ui/react-avatar": "latest", + "@radix-ui/react-checkbox": "latest", + "@radix-ui/react-collapsible": "latest", + "@radix-ui/react-context-menu": "latest", + "@radix-ui/react-dialog": "latest", + "@radix-ui/react-direction": "latest", + "@radix-ui/react-dropdown-menu": "latest", + "@radix-ui/react-hover-card": "latest", + "@radix-ui/react-label": "latest", + "@radix-ui/react-navigation-menu": "latest", + "@radix-ui/react-popover": "latest", + "@radix-ui/react-portal": "latest", + "@radix-ui/react-progress": "latest", + "@radix-ui/react-radio-group": "latest", + "@radix-ui/react-scroll-area": "latest", + "@radix-ui/react-select": "latest", + "@radix-ui/react-separator": "latest", + "@radix-ui/react-slider": "latest", + "@radix-ui/react-slot": "latest", + "@radix-ui/react-switch": "latest", + "@radix-ui/react-tabs": "latest", + "@radix-ui/react-toast": "latest", + "@radix-ui/react-toggle": "latest", + "@radix-ui/react-toggle-group": "latest", + "@radix-ui/react-toolbar": "latest", + "@radix-ui/react-tooltip": "latest", + "@radix-ui/react-visually-hidden": "latest" + }, + "peerDependencies": { + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + } + }, "node_modules/react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -2878,6 +5505,21 @@ "react": "^18.2.0" } }, + "node_modules/react-hook-form": { + "version": "7.51.3", + "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.51.3.tgz", + "integrity": "sha512-cvJ/wbHdhYx8aviSWh28w9ImjmVsb5Y05n1+FW786vEZQJV5STNM0pW6ujS+oiBecb0ARBxJFyAnXj9+GHXACQ==", + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/react-hook-form" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17 || ^18" + } + }, "node_modules/react-refresh": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.0.tgz", @@ -2887,6 +5529,51 @@ "node": ">=0.10.0" } }, + "node_modules/react-remove-scroll": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.5.5.tgz", + "integrity": "sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==", + "dependencies": { + "react-remove-scroll-bar": "^2.3.3", + "react-style-singleton": "^2.2.1", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.0", + "use-sidecar": "^1.1.2" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz", + "integrity": "sha512-DtSYaao4mBmX+HDo5YWYdBWQwYIQQshUV/dVxFxK+KM26Wjwp1gZ6rv6OC3oujI6Bfu6Xyg3TwK533AQutsn/g==", + "dependencies": { + "react-style-singleton": "^2.2.1", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/react-router": { "version": "6.22.3", "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.22.3.tgz", @@ -2917,6 +5604,83 @@ "react-dom": ">=16.8" } }, + "node_modules/react-style-singleton": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz", + "integrity": "sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==", + "dependencies": { + "get-nonce": "^1.0.0", + "invariant": "^2.2.4", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-toastify": { + "version": "10.0.5", + "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-10.0.5.tgz", + "integrity": "sha512-mNKt2jBXJg4O7pSdbNUfDdTsK9FIdikfsIE/yUCxbAEXl4HMyJaivrVFcn3Elvt5xvCQYhUZm+hqTIu1UXM3Pw==", + "dependencies": { + "clsx": "^2.1.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + } + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "dev": true, + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -3037,58 +5801,157 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "dependencies": { - "yallist": "^4.0.0" + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/snake-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.4.tgz", + "integrity": "sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==", + "dev": true, + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/source-map-js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" }, "engines": { - "node": ">=10" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "dependencies": { - "shebang-regex": "^3.0.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", "dev": true, "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "node_modules/string-width/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, - "node_modules/strip-ansi": { + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", @@ -3112,6 +5975,50 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/sucrase": { + "version": "3.35.0", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", + "integrity": "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "glob": "^10.3.10", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/sucrase/node_modules/glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "dev": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -3124,12 +6031,112 @@ "node": ">=4" } }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/svg-parser": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz", + "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==", + "dev": true + }, + "node_modules/swr": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.2.5.tgz", + "integrity": "sha512-QtxqyclFeAsxEUeZIYmsaQ0UjimSq1RZ9Un7I68/0ClKK/U3LoyQunwkQfJZr2fc22DfIXLNDc2wFyTEikCUpg==", + "dependencies": { + "client-only": "^0.0.1", + "use-sync-external-store": "^1.2.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/tailwind-merge": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.2.2.tgz", + "integrity": "sha512-tWANXsnmJzgw6mQ07nE3aCDkCK4QdT3ThPMCzawoYA2Pws7vSTCvz3Vrjg61jVUGfFZPJzxEP+NimbcW+EdaDw==", + "dependencies": { + "@babel/runtime": "^7.24.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/dcastil" + } + }, + "node_modules/tailwindcss": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.3.tgz", + "integrity": "sha512-U7sxQk/n397Bmx4JHbJx/iSOOv5G+II3f1kpLpY2QeUv5DcPdcTsYLlusZfq1NthHS1c1cZoyFmmkex1rzke0A==", + "dev": true, + "dependencies": { + "@alloc/quick-lru": "^5.2.0", + "arg": "^5.0.2", + "chokidar": "^3.5.3", + "didyoumean": "^1.2.2", + "dlv": "^1.1.3", + "fast-glob": "^3.3.0", + "glob-parent": "^6.0.2", + "is-glob": "^4.0.3", + "jiti": "^1.21.0", + "lilconfig": "^2.1.0", + "micromatch": "^4.0.5", + "normalize-path": "^3.0.0", + "object-hash": "^3.0.0", + "picocolors": "^1.0.0", + "postcss": "^8.4.23", + "postcss-import": "^15.1.0", + "postcss-js": "^4.0.1", + "postcss-load-config": "^4.0.1", + "postcss-nested": "^6.0.1", + "postcss-selector-parser": "^6.0.11", + "resolve": "^1.22.2", + "sucrase": "^3.32.0" + }, + "bin": { + "tailwind": "lib/cli.js", + "tailwindcss": "lib/cli.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -3163,6 +6170,17 @@ "typescript": ">=4.2.0" } }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true + }, + "node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -3210,7 +6228,6 @@ "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==", - "dev": true, "funding": [ { "type": "opencollective", @@ -3245,6 +6262,55 @@ "punycode": "^2.1.0" } }, + "node_modules/use-callback-ref": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.2.tgz", + "integrity": "sha512-elOQwe6Q8gqZgDA8mrh44qRTQqpIHDcZ3hXTLjBe1i4ph8XpNJnO+aQf3NaG+lriLopI4HMx9VjQLfPQ6vhnoA==", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.2.tgz", + "integrity": "sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "^16.9.0 || ^17.0.0 || ^18.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sync-external-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", + "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/util": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", @@ -3253,11 +6319,29 @@ "inherits": "2.0.3" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, "node_modules/util/node_modules/inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vite": { "version": "5.2.8", "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.8.tgz", @@ -3313,6 +6397,20 @@ } } }, + "node_modules/vite-plugin-svgr": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/vite-plugin-svgr/-/vite-plugin-svgr-4.2.0.tgz", + "integrity": "sha512-SC7+FfVtNQk7So0XMjrrtLAbEC8qjFPifyD7+fs/E6aaNdVde6umlVVh0QuwDLdOMu7vp5RiGFsB70nj5yo0XA==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "^5.0.5", + "@svgr/core": "^8.1.0", + "@svgr/plugin-jsx": "^8.1.0" + }, + "peerDependencies": { + "vite": "^2.6.0 || 3 || 4 || 5" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3328,6 +6426,133 @@ "node": ">= 8" } }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3340,6 +6565,18 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true }, + "node_modules/yaml": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.1.tgz", + "integrity": "sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg==", + "dev": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -3351,6 +6588,14 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.23.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.4.tgz", + "integrity": "sha512-/AtWOKbBgjzEYYQRNfoGKHObgfAZag6qUJX1VbHo2PRBgS+wfWagEY2mizjfyAPcGesrJOcx/wcl0L9WnVrHFw==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/src/web/package.json similarity index 53% rename from package.json rename to src/web/package.json index 3c9b1425..25bd96af 100644 --- a/package.json +++ b/src/web/package.json @@ -10,22 +10,43 @@ "preview": "vite preview" }, "dependencies": { + "@hookform/resolvers": "^3.3.4", + "@react-oauth/google": "^0.12.1", + "@tanstack/react-table": "^8.16.0", + "autoprefixer": "^10.4.19", + "axios": "^1.6.8", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.0", + "date-fns": "^3.6.0", + "dotenv": "^16.4.5", + "lucide-react": "^0.367.0", "path": "^0.12.7", + "postcss": "^8.4.38", + "radix-ui": "^1.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "react-router-dom": "^6.22.3" + "react-hook-form": "^7.51.3", + "react-router-dom": "^6.22.3", + "react-toastify": "^10.0.5", + "swr": "^2.2.5", + "tailwind-merge": "^2.2.2", + "uuid": "^9.0.1", + "zod": "^3.23.4" }, "devDependencies": { "@types/node": "^20.12.6", "@types/react": "^18.2.66", "@types/react-dom": "^18.2.22", + "@types/uuid": "^9.0.8", "@typescript-eslint/eslint-plugin": "^7.2.0", "@typescript-eslint/parser": "^7.2.0", "@vitejs/plugin-react": "^4.2.1", "eslint": "^8.57.0", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-react-refresh": "^0.4.6", + "tailwindcss": "^3.4.3", "typescript": "^5.2.2", - "vite": "^5.2.0" + "vite": "^5.2.0", + "vite-plugin-svgr": "^4.2.0" } } diff --git a/src/web/postcss.config.js b/src/web/postcss.config.js new file mode 100644 index 00000000..2e7af2b7 --- /dev/null +++ b/src/web/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/public/vite.svg b/src/web/public/vite.svg similarity index 100% rename from public/vite.svg rename to src/web/public/vite.svg diff --git a/src/web/src/assets/svgs/add-icon.svg b/src/web/src/assets/svgs/add-icon.svg new file mode 100644 index 00000000..985291e8 --- /dev/null +++ b/src/web/src/assets/svgs/add-icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/web/src/assets/svgs/chat-square.svg b/src/web/src/assets/svgs/chat-square.svg new file mode 100644 index 00000000..b3e011f1 --- /dev/null +++ b/src/web/src/assets/svgs/chat-square.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/web/src/assets/svgs/cognix-sm.svg b/src/web/src/assets/svgs/cognix-sm.svg new file mode 100644 index 00000000..28cbde97 --- /dev/null +++ b/src/web/src/assets/svgs/cognix-sm.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/web/src/assets/svgs/cognix.svg b/src/web/src/assets/svgs/cognix.svg new file mode 100644 index 00000000..4c68c615 --- /dev/null +++ b/src/web/src/assets/svgs/cognix.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/web/src/assets/svgs/config.svg b/src/web/src/assets/svgs/config.svg new file mode 100644 index 00000000..68912883 --- /dev/null +++ b/src/web/src/assets/svgs/config.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/web/src/assets/svgs/connectors.svg b/src/web/src/assets/svgs/connectors.svg new file mode 100644 index 00000000..d61c8675 --- /dev/null +++ b/src/web/src/assets/svgs/connectors.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/web/src/assets/svgs/copy-icon.svg b/src/web/src/assets/svgs/copy-icon.svg new file mode 100644 index 00000000..2f3011f8 --- /dev/null +++ b/src/web/src/assets/svgs/copy-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/document.svg b/src/web/src/assets/svgs/document.svg new file mode 100644 index 00000000..f8d0dd15 --- /dev/null +++ b/src/web/src/assets/svgs/document.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/edit-icon.svg b/src/web/src/assets/svgs/edit-icon.svg new file mode 100644 index 00000000..cbf0237d --- /dev/null +++ b/src/web/src/assets/svgs/edit-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/embedding.svg b/src/web/src/assets/svgs/embedding.svg new file mode 100644 index 00000000..f0afface --- /dev/null +++ b/src/web/src/assets/svgs/embedding.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/feedback.svg b/src/web/src/assets/svgs/feedback.svg new file mode 100644 index 00000000..e8405268 --- /dev/null +++ b/src/web/src/assets/svgs/feedback.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/file-icon.svg b/src/web/src/assets/svgs/file-icon.svg new file mode 100644 index 00000000..95522e87 --- /dev/null +++ b/src/web/src/assets/svgs/file-icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/web/src/assets/svgs/file-white-icon.svg b/src/web/src/assets/svgs/file-white-icon.svg new file mode 100644 index 00000000..515e9867 --- /dev/null +++ b/src/web/src/assets/svgs/file-white-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/filter-arrow-active.svg b/src/web/src/assets/svgs/filter-arrow-active.svg new file mode 100644 index 00000000..9303ffb5 --- /dev/null +++ b/src/web/src/assets/svgs/filter-arrow-active.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/filter-arrow-unactive.svg b/src/web/src/assets/svgs/filter-arrow-unactive.svg new file mode 100644 index 00000000..4f937f01 --- /dev/null +++ b/src/web/src/assets/svgs/filter-arrow-unactive.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/google.svg b/src/web/src/assets/svgs/google.svg new file mode 100644 index 00000000..5ac7a97b --- /dev/null +++ b/src/web/src/assets/svgs/google.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/llm.svg b/src/web/src/assets/svgs/llm.svg new file mode 100644 index 00000000..f49de179 --- /dev/null +++ b/src/web/src/assets/svgs/llm.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/web/src/assets/svgs/logout-icon.svg b/src/web/src/assets/svgs/logout-icon.svg new file mode 100644 index 00000000..54583228 --- /dev/null +++ b/src/web/src/assets/svgs/logout-icon.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/web/src/assets/svgs/menu.svg b/src/web/src/assets/svgs/menu.svg new file mode 100644 index 00000000..61e3f398 --- /dev/null +++ b/src/web/src/assets/svgs/menu.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/pause-icon.svg b/src/web/src/assets/svgs/pause-icon.svg new file mode 100644 index 00000000..8a5fc255 --- /dev/null +++ b/src/web/src/assets/svgs/pause-icon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/web/src/assets/svgs/send-icon.svg b/src/web/src/assets/svgs/send-icon.svg new file mode 100644 index 00000000..b7fc3a71 --- /dev/null +++ b/src/web/src/assets/svgs/send-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/sidebar-closed-icon.svg b/src/web/src/assets/svgs/sidebar-closed-icon.svg new file mode 100644 index 00000000..e848e930 --- /dev/null +++ b/src/web/src/assets/svgs/sidebar-closed-icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/web/src/assets/svgs/sidebar-icon.svg b/src/web/src/assets/svgs/sidebar-icon.svg new file mode 100644 index 00000000..bb37e050 --- /dev/null +++ b/src/web/src/assets/svgs/sidebar-icon.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/web/src/assets/svgs/thumb-up.svg b/src/web/src/assets/svgs/thumb-up.svg new file mode 100644 index 00000000..d36626e4 --- /dev/null +++ b/src/web/src/assets/svgs/thumb-up.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/thumn-down.svg b/src/web/src/assets/svgs/thumn-down.svg new file mode 100644 index 00000000..c75935b5 --- /dev/null +++ b/src/web/src/assets/svgs/thumn-down.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/trash-icon.svg b/src/web/src/assets/svgs/trash-icon.svg new file mode 100644 index 00000000..d3748486 --- /dev/null +++ b/src/web/src/assets/svgs/trash-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/web/src/assets/svgs/users.svg b/src/web/src/assets/svgs/users.svg new file mode 100644 index 00000000..18649b21 --- /dev/null +++ b/src/web/src/assets/svgs/users.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/web/src/components/DisplayTabs.tsx b/src/web/src/components/DisplayTabs.tsx new file mode 100644 index 00000000..f34955ed --- /dev/null +++ b/src/web/src/components/DisplayTabs.tsx @@ -0,0 +1,53 @@ +import * as React from 'react'; +import * as TabsPrimitive from '@radix-ui/react-tabs'; + +import { cn } from '@/lib/utils'; + +const Tabs = TabsPrimitive.Root; + +const TabsList = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +TabsList.displayName = TabsPrimitive.List.displayName; + +const TabsTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +TabsTrigger.displayName = TabsPrimitive.Trigger.displayName; + +const TabsContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +TabsContent.displayName = TabsPrimitive.Content.displayName; + +export { Tabs, TabsList, TabsTrigger, TabsContent }; diff --git a/src/web/src/components/MenuTabs/index.tsx b/src/web/src/components/MenuTabs/index.tsx new file mode 100644 index 00000000..3228a982 --- /dev/null +++ b/src/web/src/components/MenuTabs/index.tsx @@ -0,0 +1,64 @@ +import * as React from 'react'; +import * as TabsPrimitive from '@radix-ui/react-tabs'; + +import { cn } from '@/lib/utils'; + +const Tabs = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + {props.children} + +)); + +const TabsList = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +TabsList.displayName = TabsPrimitive.List.displayName; + +const TabsTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +
{props.children}
+ +)); +TabsTrigger.displayName = TabsPrimitive.Trigger.displayName; + +const TabsContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +TabsContent.displayName = TabsPrimitive.Content.displayName; + +export { Tabs, TabsList, TabsTrigger, TabsContent }; diff --git a/src/web/src/components/chat/card-grid.tsx b/src/web/src/components/chat/card-grid.tsx new file mode 100644 index 00000000..326d26c4 --- /dev/null +++ b/src/web/src/components/chat/card-grid.tsx @@ -0,0 +1,34 @@ +import React, { Dispatch, memo, SetStateAction } from "react"; +import { Card } from "../ui/card"; +import { Persona } from "@/models/settings"; + +interface Props { + chunkArray: (array: any[], size: number) => any[]; + personas: Persona[]; + selectedPersona: string; + setSelectedPersona: Dispatch>; +} + +export const CardGrid = memo( + ({ chunkArray, personas, selectedPersona, setSelectedPersona }: Props) => { + return ( +
+
+ {chunkArray(personas, 2).map((chunk, chunkIndex) => ( + + {chunk.map((persona: Persona) => ( + setSelectedPersona(persona.id)} + /> + ))} + + ))} +
+
+ ); + } +); diff --git a/src/web/src/components/chat/chat-input.tsx b/src/web/src/components/chat/chat-input.tsx new file mode 100644 index 00000000..26df330f --- /dev/null +++ b/src/web/src/components/chat/chat-input.tsx @@ -0,0 +1,46 @@ +import React, { memo } from "react"; +import { Button } from "../ui/button"; +import { Input } from "../ui/input"; +import SendIcon from "@/assets/svgs/send-icon.svg?react"; + +interface ChatInputProps { + onSubmit: () => Promise; + isDeactivateSendingButton: boolean; + textInputRef: any; +} + +export const ChatInput: React.FC = memo( + ({ onSubmit, isDeactivateSendingButton, textInputRef }: ChatInputProps) => { + return ( +
+
+ { + if (event.key === "Enter" && !isDeactivateSendingButton) { + onSubmit(); + } + }} + /> + +
+
+ + CogniX can make mistakes. Consider checking critical information. + +
+
+ ); + } +); diff --git a/src/web/src/components/chat/components/document-card.tsx b/src/web/src/components/chat/components/document-card.tsx new file mode 100644 index 00000000..7fd1b49f --- /dev/null +++ b/src/web/src/components/chat/components/document-card.tsx @@ -0,0 +1,55 @@ +import React, { useEffect, useState } from "react"; +import { formatDistanceToNow } from "date-fns"; + +export interface DocumentProps { + id: string; + link: string; + content: string; + document_id: string; + className?: string; + date: string; +} +const DocumentCard: React.FC = ({ link, content, date }) => { + const [timeElapsed, setTimeElapsed] = useState(""); + + useEffect(() => { + const parsedDate = new Date(date); + + const updateTimeElapsed = () => { + setTimeElapsed(formatDistanceToNow(parsedDate, { addSuffix: true })); + }; + + updateTimeElapsed(); + + const intervalId = setInterval(updateTimeElapsed, 60000); + + return () => clearInterval(intervalId); + }, [date]); + + return ( +
+
+
+
+

+ {link.length < 16 ? link : `${link.slice(0, 16)}...`} +

+

+ Updated {timeElapsed} +

+
+
+
+
+
+ AI Chat interactions, the client could indicate to NATS that multiple + replies should be allowed. This flexibility allows for more dynamic + and scalable communication between clients and servers + {content} +
+
+
+ ); +}; + +export default DocumentCard; diff --git a/src/web/src/components/chat/components/message-card.tsx b/src/web/src/components/chat/components/message-card.tsx new file mode 100644 index 00000000..f403927c --- /dev/null +++ b/src/web/src/components/chat/components/message-card.tsx @@ -0,0 +1,124 @@ +import React, { useContext, useState } from "react"; +import BotIcon from "@/assets/svgs/cognix-sm.svg?react"; +import CopyIcon from "@/assets/svgs/copy-icon.svg?react"; +import ThumbUpIcon from "@/assets/svgs/thumb-up.svg?react"; +import ThumbDownIcon from "@/assets/svgs/thumn-down.svg?react"; +import FileWhiteIcon from "@/assets/svgs/file-white-icon.svg?react"; +import axios from "axios"; +import { AuthContext } from "@/context/AuthContext"; +import { Document, MessageFeedback } from "@/models/chat"; + +export interface MessageProps { + id: string; + sender?: string; + message: string; + timestamp: string; + className?: string; + citations?: Document[]; + isResponse?: boolean; + feedback?: MessageFeedback; +} + +const MessageCard: React.FC = ({ + id, + sender, + message, + citations, + feedback, + isResponse, + className, +}) => { + const { firstName, lastName } = useContext(AuthContext); + + const [feedbackValue, setFeedbackValue] = useState(feedback?.up_votes); + + async function feedbackMessage(vote: "upvote" | "downvote"): Promise { + await axios + .post(import.meta.env.VITE_PLATFORM_API_CHAT_FEEDBACK_MESSAGE_URL, { + id: id, + vote: vote, + }) + .then((response) => { + if (response.status == 200) { + setFeedbackValue(vote === "upvote"); + } + }); + } + + return ( +
+
+
+ {isResponse ? ( + + ) : ( +
+ + {firstName && `${firstName.charAt(0)}`} + + + {lastName && `${lastName.charAt(0)}`} + +
+ )} +
+
{sender}
+
+
+
+
{message}
+
+ {sender !== "You" &&
Sources:
} + {citations?.map((citation) => ( +
+ + {citation.link} +
+ ))} +
+ {isResponse && ( +
+
{ + navigator.clipboard.writeText(message); + }} + > + +
+
{ + feedbackMessage("upvote"); + }} + > + +
+
{ + feedbackMessage("downvote"); + }} + > + +
+
+ )} +
+
+
+ ); +}; + +export default MessageCard; diff --git a/src/web/src/components/chat/components/retrieved-knowledge.tsx b/src/web/src/components/chat/components/retrieved-knowledge.tsx new file mode 100644 index 00000000..8a726252 --- /dev/null +++ b/src/web/src/components/chat/components/retrieved-knowledge.tsx @@ -0,0 +1,52 @@ +import { memo } from "react"; +import DocumentCard from "./document-card"; +import FileIcon from "@/assets/svgs/file-icon.svg?react"; +import { ChatMessage } from "@/models/chat"; + +interface Props { + messages: ChatMessage[]; + withHeader?: boolean; +} + +export const RetrievedKnowledge = memo(({ messages, withHeader }: Props) => { + + return ( +
+ {withHeader && ( +
+ + Retrieved Knowledge +
+ )} + {messages && messages.length ? ( +
+ {messages?.map((message) => + message.citations?.map((citation, index) => ( + + )) + )} +
+ ) : ( +
+
+ + When you run ask a question, the + +
+
+ + retrieved knowledge will show up here + +
+
+ )} +
+ ); +}); diff --git a/src/web/src/components/chat/index-1.tsx b/src/web/src/components/chat/index-1.tsx new file mode 100644 index 00000000..54f801d8 --- /dev/null +++ b/src/web/src/components/chat/index-1.tsx @@ -0,0 +1,262 @@ +import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import axios from "axios"; +import { ChatMessage } from "@/models/chat"; +import { useParams } from "react-router-dom"; +import { v4 as uuidv4 } from "uuid"; +import { Persona } from "@/models/settings"; +import { router } from "@/main"; +import { toast } from "react-toastify"; +import { RetrievedKnowledge } from "./components/retrieved-knowledge"; +import { useMessages } from "@/context/ChatContext"; +import { ChatInput } from "./chat-input"; +import MessagesList from "./messages-list"; +import { PersonaSelection } from "./persona-list-section"; + +export function ChatComponent() { + const { messages, setMessages, addMessage } = useMessages(); + const [personas, setPersonas] = useState([]); + const [selectedPersona, setSelectedPersona] = useState(""); + const [newMessage, setNewMessage] = useState(null); + const [isDeactivateSendingButton, setIsDeactivateSendingButton] = + useState(false); + const textInputRef = useRef(null); + + const { chatId } = useParams<{ chatId?: string }>(); + + async function onHandleSubmit() { + if (textInputRef.current?.value) { + setIsDeactivateSendingButton(true); + chatId + ? await createMessages(textInputRef.current?.value ?? "") + : await createChat(textInputRef.current?.value ?? ""); + setIsDeactivateSendingButton(false); + } + } + + async function getMessages(): Promise { + try { + const response = await axios.get( + `${import.meta.env.VITE_PLATFORM_API_CHAT_DETAIL_URL}/${chatId}` + ); + if (response.status === 200) { + setMessages(response.data.data.messages); + } else { + setMessages([]); + } + } catch (error) { + console.error("Error fetching messages:", error); + } + } + + async function createChat(text: string) { + if (!selectedPersona) { + if (!personas || personas.length === 0) { + toast.error("You don't have any available Assistants yet."); + return; + } + } + try { + const response = await axios.post( + import.meta.env.VITE_PLATFORM_API_CHAT_CREATE_URL, + { + description: text, + one_shot: true, + persona_id: selectedPersona || personas[0].id, + } + ); + if (response.status === 201) { + await createMessages(text, response.data.data.id); + } else { + setPersonas([]); + } + } catch (error) { + console.error("Error creating chat:", error); + } + } + + async function createMessages(text: string, passedChatId?: string): Promise { + const currentChatId = chatId ?? passedChatId; + if (!currentChatId) { + console.error("No chat ID available"); + return; + } + + const userMessage: ChatMessage = { + id: uuidv4(), + message: text, + chat_session_id: currentChatId, + message_type: "user", + time_sent: new Date().toString(), + }; + + try { + const response = await fetch( + `${import.meta.env.VITE_PLATFORM_API_CHAT_SEND_MESSAGE_URL}`, + { + method: "POST", + body: JSON.stringify({ + message: text, + chat_session_id: currentChatId, + }), + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${JSON.parse( + String(localStorage.getItem("access_token")) + )}`, + }, + } + ); + + if (!response.ok || !response.body) { + throw new Error(response.statusText); + } + + addMessage(userMessage); + textInputRef.current!.value = ""; + + const reader = response.body.getReader(); + const decoder = new TextDecoder('utf-8'); + let messageContent = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + messageContent += chunk; + processChunk(chunk, currentChatId); + } + } catch (error) { + console.error("Error creating messages:", error); + } + } + + function processChunk(chunk: string, chatSessionId: string) { + const streams = chunk.split("\n"); + + streams.forEach(stream => { + if (stream) { + const [event, data] = stream.split("data:"); + if (event === "event:document") { + const doc = JSON.parse(data); + updateDocumentMessage(doc.Document); + } else if (event === "event:message") { + const response = JSON.parse(data); + addMessage({ ...response.Message, message: "" }); + setNewMessage(response.Message); + } else if (event === "event:error") { + const response = JSON.parse(data); + router.navigate(`/chat/${chatSessionId}`); + toast.error(response.Message.error); + } + } + }); + } + + function updateDocumentMessage(document: any) { + setMessages(prev => { + const messageIndex = prev.findIndex( + (message) => message.id === document.message_id + ); + + if (messageIndex !== -1) { + const updatedMessages = [...prev]; + updatedMessages[messageIndex] = { + ...updatedMessages[messageIndex], + citations: ( + updatedMessages[messageIndex].citations || [] + ).concat(document), + }; + + return updatedMessages; + } + + return prev; + }); + } + + async function getPersonas() { + try { + const response = await axios.get( + import.meta.env.VITE_PLATFORM_API_LLM_LIST_URL + ); + if (response.status === 200) { + setPersonas(response.data.data); + } else { + setPersonas([]); + } + } catch (error) { + console.error("Error fetching personas:", error); + setPersonas([]); + } + } + + function chunkArray(array: any[], size: number) { + const chunkedArray = []; + for (let i = 0; i < array.length; i += size) { + chunkedArray.push(array.slice(i, i + size)); + } + return chunkedArray; + } + + useLayoutEffect(() => { + if (chatId) { + getMessages(); + } else { + getPersonas(); + setMessages([]); + } + }, [chatId]); + + useEffect(() => { + let index = 0; + const intervalId = setInterval(() => { + if (newMessage && newMessage.message) { + setMessages((prevMessages) => + prevMessages?.map((prevMessage) => + prevMessage.id === newMessage.id + ? { + ...prevMessage, + message: newMessage.message.substr(0, index + 1), + } + : prevMessage + ) + ); + index++; + if (index >= newMessage.message.length) { + clearInterval(intervalId); + } + } + }, 25); + if (!chatId && newMessage) { + router.navigate(`/chat/${newMessage.chat_session_id}`); + } + return () => { + clearInterval(intervalId); + }; + }, [newMessage]); + + return ( +
+
+ {messages.length === 0 ? ( + + ) : ( + + )} + +
+
+ +
+
+ ); +} diff --git a/src/web/src/components/chat/index-2.tsx b/src/web/src/components/chat/index-2.tsx new file mode 100644 index 00000000..7fdd04d8 --- /dev/null +++ b/src/web/src/components/chat/index-2.tsx @@ -0,0 +1,236 @@ +import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import axios from "axios"; +import { ChatMessage } from "@/models/chat"; +import { useParams } from "react-router-dom"; +import { v4 as uuidv4 } from "uuid"; +import { Persona } from "@/models/settings"; +import { router } from "@/main"; +import { toast } from "react-toastify"; +import { RetrievedKnowledge } from "./components/retrieved-knowledge"; +import { useMessages } from "@/context/ChatContext"; +import { ChatInput } from "./chat-input"; +import MessagesList from "./messages-list"; +import { PersonaSelection } from "./persona-list-section"; + +const API_BASE_URL = import.meta.env.VITE_PLATFORM_API_BASE_URL; + +export function ChatComponent() { + const { messages, setMessages, addMessage } = useMessages(); + const [personas, setPersonas] = useState([]); + const [selectedPersona, setSelectedPersona] = useState(""); + const [newMessage, setNewMessage] = useState(null); + const [isDeactivateSendingButton, setIsDeactivateSendingButton] = useState(false); + const textInputRef = useRef(null); + + const { chatId } = useParams<{ chatId?: string }>(); + + useLayoutEffect(() => { + if (chatId) { + fetchMessages(); + } else { + fetchPersonas(); + setMessages([]); + } + }, [chatId]); + + useEffect(() => { + if (newMessage && newMessage.message) { + const intervalId = setInterval(() => { + updateMessageContent(); + }, 25); + + return () => clearInterval(intervalId); + } + }, [newMessage]); + + async function fetchMessages() { + try { + const response = await axios.get(`${API_BASE_URL}/chat/detail/${chatId}`); + if (response.status === 200) { + setMessages(response.data.data.messages); + } else { + setMessages([]); + } + } catch (error) { + console.error("Error fetching messages:", error); + } + } + + async function fetchPersonas() { + try { + const response = await axios.get(`${API_BASE_URL}/llm/list`); + if (response.status === 200) { + setPersonas(response.data.data); + } else { + setPersonas([]); + } + } catch (error) { + console.error("Error fetching personas:", error); + setPersonas([]); + } + } + + async function createChat(text: string) { + if (!selectedPersona) { + if (!personas.length) { + toast.error("You don't have any available Assistants yet."); + return; + } + } + + try { + const response = await axios.post(`${API_BASE_URL}/chat/create`, { + description: text, + one_shot: true, + persona_id: selectedPersona || personas[0].id, + }); + + if (response.status === 201) { + await createMessages(text, response.data.data.id); + } + } catch (error) { + console.error("Error creating chat:", error); + } + } + + async function createMessages(text: string, passedChatId?: string) { + const currentChatId = chatId || passedChatId; + if (!currentChatId) { + console.error("No chat ID available"); + return; + } + + const userMessage: ChatMessage = { + id: uuidv4(), + message: text, + chat_session_id: currentChatId, + message_type: "user", + time_sent: new Date().toISOString(), + }; + + try { + const response = await fetch(`${API_BASE_URL}/chat/send-message`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${JSON.parse(localStorage.getItem("access_token") as string)}`, + }, + body: JSON.stringify({ message: text, chat_session_id: currentChatId }), + }); + + if (!response.ok || !response.body) { + throw new Error(response.statusText); + } + + addMessage(userMessage); + textInputRef.current!.value = ""; + + const reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8"); + let messageContent = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + messageContent += chunk; + processStreamedChunk(chunk, currentChatId); + } + } catch (error) { + console.error("Error creating messages:", error); + } + } + + function processStreamedChunk(chunk: string, chatSessionId: string) { + const streams = chunk.split("\n"); + + streams.forEach(stream => { + if (stream) { + const [event, data] = stream.split("data:"); + if (event === "event:document") { + const doc = JSON.parse(data); + updateDocumentMessage(doc.Document); + } else if (event === "event:message") { + const response = JSON.parse(data); + addMessage({ ...response.Message, message: "" }); + setNewMessage(response.Message); + } else if (event === "event:error") { + const response = JSON.parse(data); + router.navigate(`/chat/${chatSessionId}`); + toast.error(response.Message.error); + } + } + }); + } + + function updateDocumentMessage(document: any) { + setMessages(prev => { + const messageIndex = prev.findIndex( + message => message.id === document.message_id + ); + + if (messageIndex !== -1) { + const updatedMessages = [...prev]; + updatedMessages[messageIndex] = { + ...updatedMessages[messageIndex], + citations: (updatedMessages[messageIndex].citations || []).concat(document), + }; + + return updatedMessages; + } + + return prev; + }); + } + + function updateMessageContent() { + setMessages(prevMessages => + prevMessages.map(prevMessage => + prevMessage.id === newMessage!.id + ? { ...prevMessage, message: newMessage!.message.substring(0, newMessage!.message.length) } + : prevMessage + ) + ); + } + + async function handleSubmit() { + if (textInputRef.current?.value) { + setIsDeactivateSendingButton(true); + chatId ? await createMessages(textInputRef.current.value) : await createChat(textInputRef.current.value); + setIsDeactivateSendingButton(false); + } + } + + function chunkArray(array: any[], size: number) { + const chunkedArray = []; + for (let i = 0; i < array.length; i += size) { + chunkedArray.push(array.slice(i, i + size)); + } + return chunkedArray; + } + + return ( +
+
+ {messages.length === 0 ? ( + + ) : ( + + )} + +
+
+ +
+
+ ); +} diff --git a/src/web/src/components/chat/index.tsx b/src/web/src/components/chat/index.tsx new file mode 100644 index 00000000..d6ddf1ea --- /dev/null +++ b/src/web/src/components/chat/index.tsx @@ -0,0 +1,263 @@ +import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import axios from "axios"; +import { ChatMessage } from "@/models/chat"; +import { useParams } from "react-router-dom"; +import { v4 as uuidv4 } from "uuid"; +import { Persona } from "@/models/settings"; +import { router } from "@/main"; +import { toast } from "react-toastify"; +import { RetrievedKnowledge } from "./components/retrieved-knowledge"; +import { useMessages } from "@/context/ChatContext"; +import { ChatInput } from "./chat-input"; +import MessagesList from "./messages-list"; +import { PersonaSelection } from "./persona-list-section"; + +export function ChatComponent() { + const { messages, setMessages, addMessage } = useMessages(); + const [personas, setPersonas] = useState([]); + const [selectedPersona, setSelectedPersona] = useState(""); + const [newMessage, setNewMessage] = useState(); + const [isDeactivateSendingButton, setIsDeactivateSendingButton] = + useState(false); + const textInputRef = useRef(null); + + const { chatId } = useParams<{ + chatId?: string; + }>(); + + async function onHandleSubmit() { + if (textInputRef.current?.value) { + setIsDeactivateSendingButton(true); + chatId + ? await createMessages(textInputRef.current?.value ?? "") + : await createChat(textInputRef.current?.value ?? ""); + setIsDeactivateSendingButton(false); + } + } + + async function getMessages(): Promise { + await axios + .get(`${import.meta.env.VITE_PLATFORM_API_CHAT_DETAIL_URL}/${chatId}`) + .then(function (response) { + if (response.status == 200) { + setMessages(response.data.data.messages); + } else { + setMessages([]); + } + }) + .catch(function (error) { + console.error("Error fetching messages:", error); + }); + } + + async function createChat(text: string) { + if (!selectedPersona) { + if (!personas || personas.length === 0) { + toast.error("You don't have any available Assistants yet."); + return; + } + } + await axios + .post(import.meta.env.VITE_PLATFORM_API_CHAT_CREATE_URL, { + description: text, + one_shot: true, + persona_id: selectedPersona || personas[0].id, + }) + .then(async function (response) { + if (response.status == 201) { + await createMessages(text, response.data.data.id); + } else { + setPersonas([]); + } + }); + } + + async function createMessages( + text: string, + passedChatId?: string + ): Promise { + try { + const currentChatId = chatId ?? passedChatId; + if (!currentChatId) { + throw new Error("Chat ID is not provided."); + } + const userMessage: ChatMessage = { + id: uuidv4(), + message: text, + chat_session_id: currentChatId, + message_type: "user", + time_sent: new Date().toISOString(), + }; + const response = await fetch( + `${import.meta.env.VITE_PLATFORM_API_CHAT_SEND_MESSAGE_URL}`, + { + method: "POST", + body: JSON.stringify({ + message: text, + chat_session_id: currentChatId, + }), + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${JSON.parse( + String(localStorage.getItem("access_token")) + )}`, + }, + } + ); + if (!response.ok) { + throw new Error(`Failed to send message: ${response.statusText}`); + } + addMessage(userMessage); + if (textInputRef?.current) { + textInputRef.current.value = ""; + } + const reader = response.body + ?.pipeThrough(new TextDecoderStream()) + .getReader(); + + if (!reader) { + throw new Error("Failed to get reader from response body."); + } + + let buffer = ""; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + buffer += value; + const events = buffer.split("\n\n"); + buffer = events.pop() || ""; + for (const event of events) { + const [eventLine, dataLine] = event.split("\n"); + if (!eventLine || !dataLine) continue; + const eventType = eventLine.split(":")[1]?.trim(); + const data = dataLine.split("data:")[1]?.trim(); + if (!eventType || !data) { + console.warn("Invalid event or data:", { eventLine, dataLine }); + continue; + } + try { + const parsedData = JSON.parse(data); + if (eventType === "document") { + const doc = parsedData; + setMessages((prev) => { + const messageIndex = prev.findIndex( + (message) => message.id === doc.Document.message_id + ); + if (messageIndex !== -1) { + const updatedMessages = [...prev]; + updatedMessages[messageIndex] = { + ...updatedMessages[messageIndex], + citations: ( + updatedMessages[messageIndex].citations || [] + ).concat(doc.Document), + }; + return updatedMessages; + } + return prev; + }); + } else if (eventType === "message") { + setMessages((prev) => [ + ...(prev ?? []), + { ...parsedData.Message, message: "" }, + ]); + setNewMessage(parsedData.Message); + } else if (eventType === "error") { + router.navigate(`/chat/${currentChatId}`); + toast.error(parsedData.Message.error); + } + } catch (error) { + console.error("Error parsing JSON:", error, "Data:", data); + } + } + } + } catch (error) { + console.error("Error in createMessages:", error); + } + } + + async function getPersonas() { + try { + const response = await axios.get( + import.meta.env.VITE_PLATFORM_API_LLM_LIST_URL + ); + if (response.status === 200) { + setPersonas(response.data.data); + } else { + setPersonas([]); + } + } catch (error) { + console.error("Error fetching personas:", error); + setPersonas([]); + } + } + + function chunkArray(array: any[], size: number) { + const chunkedArray = []; + for (let i = 0; i < array.length; i += size) { + chunkedArray.push(array.slice(i, i + size)); + } + return chunkedArray; + } + + useLayoutEffect(() => { + if (chatId) { + getMessages(); + } else { + getPersonas(); + setMessages([]); + } + }, [chatId]); + + useEffect(() => { + let index = 0; + const intervalId = setInterval(() => { + if (newMessage && newMessage.message) { + setMessages((prevMessages) => + prevMessages?.map((prevMessage) => + prevMessage.id === newMessage.id + ? { + ...prevMessage, + message: newMessage.message.substr(0, index + 1), + } + : prevMessage + ) + ); + index++; + if (index >= newMessage.message.length) { + clearInterval(intervalId); + } + } + }, 25); + if (!chatId && newMessage) { + router.navigate(`/chat/${newMessage.chat_session_id}`); + } + return () => { + clearInterval(intervalId); + }; + }, [newMessage]); + + return ( +
+
+ {messages.length == 0 ? ( + + ) : ( + + )} + +
+
+ +
+
+ ); +} diff --git a/src/web/src/components/chat/messages-list.tsx b/src/web/src/components/chat/messages-list.tsx new file mode 100644 index 00000000..85fa0741 --- /dev/null +++ b/src/web/src/components/chat/messages-list.tsx @@ -0,0 +1,57 @@ +import React, { useEffect } from "react"; +import { ChatMessage } from "@/models/chat"; +import { dataConverter } from "@/lib/utils"; +import MessageCard from "./components/message-card"; + +interface MessagesListProps { + messages: ChatMessage[]; + newMessage: ChatMessage | null | undefined; +} + +const MessagesList: React.FC = ({ + messages, + newMessage, +}) => { + useEffect(() => { + let index = 0; + const intervalId = setInterval(() => { + if (newMessage && newMessage.message) { + messages = messages?.map((message) => + message.id === newMessage.id + ? { ...message, message: newMessage.message.substr(0, index + 1) } + : message + ); + index++; + if (index >= newMessage.message.length) { + return () => clearInterval(intervalId); + } + } + }, 25); + }, [newMessage, messages]); + + return ( +
+
+
+
+ {dataConverter(messages[0]?.time_sent)} +
+
+
+ {messages.map((message, index) => ( + + ))} +
+ ); +}; + +export default MessagesList; diff --git a/src/web/src/components/chat/persona-list-section.tsx b/src/web/src/components/chat/persona-list-section.tsx new file mode 100644 index 00000000..b7cadb99 --- /dev/null +++ b/src/web/src/components/chat/persona-list-section.tsx @@ -0,0 +1,42 @@ +import React, { memo, Dispatch, SetStateAction } from "react"; +import { Persona } from "@/models/settings"; +import { CardGrid } from "./card-grid"; + +interface PersonaSelectionProps { + personas: Persona[]; + selectedPersona: string; + setSelectedPersona: Dispatch>; + chunkArray: (array: any[], size: number) => any[]; +} + +export const PersonaSelection: React.FC = memo( + ({ personas, selectedPersona, setSelectedPersona, chunkArray }) => { + return ( +
+
+ + Which assistant do you want + +
+
+ to chat with today? +
+
+ + Or ask a question immediately to use the CogniX assistant + +
+
+ {personas && ( + + )} +
+
+ ); + } +); diff --git a/src/web/src/components/dialogs/ConfirmDeleteDialog.tsx b/src/web/src/components/dialogs/ConfirmDeleteDialog.tsx new file mode 100644 index 00000000..c1d25433 --- /dev/null +++ b/src/web/src/components/dialogs/ConfirmDeleteDialog.tsx @@ -0,0 +1,50 @@ +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTrigger, + } from '@/components/ui/alert-dialog'; + import { buttonVariants } from '@/components/ui/button'; + + interface ConfirmDeleteDialogProps { + description: string; + deleteButtonText?: string; + disabled?: boolean; + onConfirm: () => void; + open?: boolean; + onOpenChange?: (open: boolean) => void; + showTrigger?: boolean; + children?: React.ReactNode; + } + + export function ConfirmDeleteDialog({ + description, + deleteButtonText, + onConfirm, + showTrigger = true, + open, + onOpenChange, + children, + }: ConfirmDeleteDialogProps) { + return ( + + {showTrigger && {children}} + + + {description} + + + Cancel + + {deleteButtonText} + + + + + ); + } + \ No newline at end of file diff --git a/src/web/src/components/dialogs/ConnectorDialog.tsx b/src/web/src/components/dialogs/ConnectorDialog.tsx new file mode 100644 index 00000000..0ee781e6 --- /dev/null +++ b/src/web/src/components/dialogs/ConnectorDialog.tsx @@ -0,0 +1,328 @@ +import { Button, buttonVariants } from "@/components/ui/button"; +import { + Dialog, + DialogClose, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { DefaultValues, useForm } from "react-hook-form"; +import { z } from "zod"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { Input } from "@/components/ui/input"; +import { useMutation } from "@/lib/mutation"; +import { toast } from "react-toastify"; +import { + CreateConnectorSchema, + UpdateConnectorSchema, +} from "@/lib/schemas/connectors"; +import { TextArea } from "../ui/textarea"; +import { Connector, SourceType } from "@/models/settings"; +import { useEffect, useState } from "react"; +import axios from "axios"; +import { capitalize } from "@/lib/utils"; +import { formSchema } from "@/lib/validations/connectors"; + +export function CreateConnectorDialog({ + defaultValues, + instance, + children, + open, + onOpenChange, +}: { + defaultValues?: DefaultValues>; + instance?: Connector; + children?: React.ReactNode; + open?: boolean; + onOpenChange: (open: boolean) => void; +}) { + const [step, setStep] = useState(1); + const [sourceTypes, setSourceTypes] = useState([]); + const [loading, setLoading] = useState(false); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + name: instance?.name || "", + connector_specific_config: instance + ? JSON.stringify( + instance.connector_specific_config ?? '{"your_key": "your_value"}' + ) + : "", + refresh_freq: instance ? String(instance.refresh_freq) : "", + source: instance?.source || "", + credential_id: instance?.credential_id || "", + ...defaultValues, + }, + }); + + const { trigger: triggerCreateConnector } = + useMutation( + import.meta.env.VITE_PLATFORM_API_CONNECTOR_CREATE_URL, + "POST" + ); + + const { trigger: triggerEditConnector } = useMutation( + `${import.meta.env.VITE_PLATFORM_API_CONNECTOR_EDIT_URL}/${instance?.id}`, + "PUT" + ); + + const onSubmit = async (values: z.infer) => { + const data = { + connector_specific_config: JSON.parse(values.connector_specific_config), + refresh_freq: Number(values.refresh_freq), + name: values.name, + source: values.source, + credential_id: values.credential_id ?? "0", + }; + if (instance) { + try { + await triggerEditConnector(data); + toast.success("Connector successfully updated"); + } catch (e) { + toast.error(e as string); + } + } else { + try { + await triggerCreateConnector(data); + toast.success("Connector successfully created"); + } catch (e) { + toast.error(e as string); + } + } + onOpenChange(false); + }; + + async function getSourceTypes() { + setLoading(true); + await axios + .get(import.meta.env.VITE_PLATFORM_API_SOURCE_TYPES_LIST_URL) + .then((response) => { + if (response.status == 200) { + setSourceTypes(response.data.data); + } + }) + .finally(() => { + setLoading(false); + }); + return []; + } + + useEffect(() => { + if (step == 1 && !instance) { + getSourceTypes(); + } + }, []); + + return ( + { + onOpenChange?.(newOpen); + form.reset(); + }} + > + {children} + + + + {instance ? "Edit Connector" : "Add Connector"} + + {!instance && ( + + Step {step}/2 + + )} + +
+ e.key === "Enter" && e.preventDefault()} + > + {step === 1 && !instance ? ( +
+ ( + <> + + + {loading ? ( +
+
+
+ ) : ( +
+ {sourceTypes.map((sourceType) => ( +
+ form.setValue("source", sourceType.id) + } + className={`bg-white p-4 rounded-sm border ${ + form.getValues("source") === sourceType.id + ? "border-primary" + : "" + } cursor-pointer`} + > +

+ {sourceType.name} +

+
+ ))} +
+ )} + + + + + + + + + + + )} + /> +
+ ) : ( + <> + ( + + Connector + + + + + + )} + /> + + { + return ( + + + + + + + ); + }} + /> + + ( + + +