From 3996f5183dad9948fccfc93333492fb85dc79fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Thu, 14 Nov 2024 13:20:28 +0100 Subject: [PATCH] merge: pull changes from `dev` (#4384) Co-authored-by: Leonardo Giacone Co-authored-by: Adrian Smijulj Co-authored-by: adrians5j --- .github/workflows/pullRequests.yml | 43 +- .github/workflows/pushDev.yml | 6 +- .github/workflows/pushNext.yml | 6 +- .github/workflows/wac/pullRequests.wac.ts | 23 +- .../wac/utils/listPackagesWithJestTests.ts | 8 +- package.json | 2 +- .../src/Operations.ts | 10 +- .../src/SynchronizationBuilder.ts | 72 ++ .../src/eventHandler.ts | 9 +- .../src/execute.ts | 9 +- .../src/executeWithRetry.ts | 10 +- .../src/index.ts | 1 + .../src/types.ts | 1 + .../__tests__/helpers/useHandler.ts | 4 + .../__tests__/mocks/context.ts | 3 + .../__tests__/mocks/store.ts | 6 +- ...ticsearchToDynamoDbSynchronization.test.ts | 172 +++++ .../dataSynchronizationTask.test.ts | 119 ++++ .../tasks/dataSynchronization/managers.ts | 40 ++ .../api-elasticsearch-tasks/jest.setup.js | 2 + packages/api-elasticsearch-tasks/package.json | 12 +- .../src/definitions/entry.ts | 4 +- .../src/definitions/table.ts | 4 +- .../src/helpers/scan.ts | 10 +- packages/api-elasticsearch-tasks/src/index.ts | 4 +- .../src/settings/IndexManager.ts | 20 +- .../src/tasks/Manager.ts | 22 +- .../createIndexes/CreateIndexesTaskRunner.ts | 8 +- .../src/tasks/createIndexes/index.ts | 11 +- .../DataSynchronizationTaskRunner.ts | 69 ++ .../dataSynchronization/createFactories.ts | 10 + .../elasticsearch/ElasticsearchFetcher.ts | 107 +++ .../elasticsearch/ElasticsearchSynchronize.ts | 101 +++ .../ElasticsearchToDynamoDbSynchronization.ts | 95 +++ .../abstractions/ElasticsearchFetcher.ts | 25 + .../abstractions/ElasticsearchSynchronize.ts | 21 + .../shouldIgnoreEsResponseError.ts | 11 + .../entities/getElasticsearchEntity.ts | 52 ++ .../entities/getElasticsearchEntityType.ts | 27 + .../dataSynchronization/entities/getTable.ts | 30 + .../dataSynchronization/entities/index.ts | 3 + .../src/tasks/dataSynchronization/index.ts | 79 +++ .../src/tasks/dataSynchronization/types.ts | 62 ++ .../EnableIndexingTaskRunner.ts | 8 +- .../src/tasks/enableIndexing/index.ts | 13 +- .../src/tasks/index.ts | 1 + .../tasks/reindexing/ReindexingTaskRunner.ts | 8 +- .../reindexing/reindexingTaskDefinition.ts | 13 +- packages/api-elasticsearch-tasks/src/types.ts | 22 +- .../tsconfig.build.json | 3 + .../api-elasticsearch-tasks/tsconfig.json | 9 + packages/api-elasticsearch/src/cursors.ts | 12 +- packages/api-elasticsearch/src/types.ts | 47 +- .../api-form-builder-so-ddb-es/src/index.ts | 21 + .../api-form-builder-so-ddb-es/src/types.ts | 22 +- .../src/handlers/eventBridgeEventHandler.ts | 5 +- .../src/plugins/createBulkActionGraphQL.ts | 10 +- .../src/plugins/createDefaultGraphQL.ts | 10 +- .../src/tasks/createEmptyTrashBinsTask.ts | 174 ++--- .../src/types.ts | 22 +- .../__tests__/__api__/setupFile.js | 19 +- packages/api-headless-cms-ddb-es/src/index.ts | 10 + packages/api-headless-cms-ddb-es/src/types.ts | 16 +- .../src/graphql/index.ts | 9 +- .../contentAPI/republish.entries.test.ts | 11 +- .../__tests__/storageOperations/context.ts | 34 + .../storageOperations/entries.test.ts | 10 +- .../fieldUniqueValues.test.ts | 10 +- packages/api-headless-cms/package.json | 2 + .../crud/contentModel/validateModelFields.ts | 1 + packages/api-headless-cms/tsconfig.build.json | 3 +- packages/api-headless-cms/tsconfig.json | 7 +- .../__tests__/__api__/setupFile.js | 20 +- .../api-page-builder-so-ddb-es/src/index.ts | 10 + .../api-page-builder-so-ddb-es/src/types.ts | 15 +- .../listPermissionsFromGroupsAndTeams.ts | 22 +- .../src/createFieldsList.ts | 17 +- .../src/entries.graphql.ts | 30 - .../src/modifiers/styles/border.ts | 8 +- packages/db-dynamodb/src/DynamoDbDriver.ts | 35 +- packages/db-dynamodb/src/types.ts | 24 - packages/db-dynamodb/src/utils/batchRead.ts | 5 +- packages/db/package.json | 3 + packages/db/src/DbRegistry.ts | 49 ++ packages/db/src/index.ts | 254 +------ packages/db/src/types.ts | 26 + packages/db/tsconfig.build.json | 2 +- packages/db/tsconfig.json | 9 +- packages/handler-db/src/index.ts | 23 +- packages/handler-db/src/types.ts | 2 +- .../src/plugins/GraphQLSchemaPlugin.ts | 9 + packages/tasks/src/runner/TaskManager.ts | 9 +- packages/tasks/src/runner/TaskRunner.ts | 2 +- .../src/runner/abstractions/TaskRunner.ts | 2 + packages/tasks/src/types.ts | 2 + yarn.lock | 669 ++++++++++-------- 96 files changed, 2137 insertions(+), 975 deletions(-) create mode 100644 packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts create mode 100644 packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts create mode 100644 packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts create mode 100644 packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts create mode 100644 packages/api-headless-cms/__tests__/storageOperations/context.ts create mode 100644 packages/db/src/DbRegistry.ts create mode 100644 packages/db/src/types.ts diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index 36ea348eab4..920fa31af13 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -43,6 +43,7 @@ jobs: run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} is-fork-pr: ${{ steps.is-fork-pr.outputs.is-fork-pr }} changed-packages: ${{ steps.detect-changed-packages.outputs.changed-packages }} + latest-webiny-version: ${{ steps.latest-webiny-version.outputs.latest-webiny-version }} steps: - uses: actions/setup-node@v4 with: @@ -79,6 +80,25 @@ jobs: .github/workflows/wac/utils/runNodeScripts/listChangedPackages.js '${{ steps.detect-changed-files.outputs.changed_files }}')" >> $GITHUB_OUTPUT + - name: Get latest Webiny version on NPM + id: latest-webiny-version + run: >- + echo "latest-webiny-version=$(npm view @webiny/cli version)" >> + $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + assignMilestone: + name: Assign milestone + needs: constants + steps: + - uses: actions/setup-node@v4 + with: + node-version: 20 + - uses: actions/checkout@v4 + - name: Print latest Webiny version + run: echo ${{ needs.constants.outputs.latest-webiny-version }} runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -118,7 +138,6 @@ jobs: staticCodeAnalysis: needs: - constants - - build name: Static code analysis steps: - uses: actions/setup-node@v4 @@ -203,8 +222,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -322,8 +341,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -394,7 +413,8 @@ jobs: '[[{"cmd":"packages/api-aco --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-aco","id":"8f23ec33f547aa62236f5c71115688d6"},{"cmd":"packages/api-audit-logs --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-audit-logs","id":"a292444cd9100f78d8fc196274393ea8"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-elasticsearch","id":"5963079c60b96202bbaf2a802ad14383"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-elasticsearch-tasks","id":"d81ad1d024a8746cc440e2e548770f8f"},{"cmd":"packages/api-file-manager --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-file-manager","id":"d6f293add4a252b96cbd770ab6e80557"},{"cmd":"packages/api-form-builder --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-form-builder","id":"3753bde0144d808eb15c755b7176386c"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -437,8 +457,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -522,7 +542,8 @@ jobs: '[[{"cmd":"packages/api-aco --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-aco","id":"e4b1b5ebc172f2657485e41c35ad1cd7"},{"cmd":"packages/api-audit-logs --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-audit-logs","id":"b36aac5f0e34dc4583e5422ae589f1ed"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-elasticsearch","id":"b0f477d6b209f654714809b318be888e"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-elasticsearch-tasks","id":"580a9577fdbd4a241034a42e1a47dee5"},{"cmd":"packages/api-file-manager --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-file-manager","id":"346430a79981d3e214c87254a08e31b2"},{"cmd":"packages/api-form-builder --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-form-builder","id":"d386cddfd3c366ad9955193dcfe74363"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -565,8 +586,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false diff --git a/.github/workflows/pushDev.yml b/.github/workflows/pushDev.yml index 04ee0cdad69..56072013d08 100644 --- a/.github/workflows/pushDev.yml +++ b/.github/workflows/pushDev.yml @@ -278,7 +278,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-aco","id":"8f23ec33f547aa62236f5c71115688d6"},{"cmd":"packages/api-audit-logs --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-audit-logs","id":"a292444cd9100f78d8fc196274393ea8"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-elasticsearch","id":"5963079c60b96202bbaf2a802ad14383"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-elasticsearch-tasks","id":"d81ad1d024a8746cc440e2e548770f8f"},{"cmd":"packages/api-file-manager --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-file-manager","id":"d6f293add4a252b96cbd770ab6e80557"},{"cmd":"packages/api-form-builder --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-form-builder","id":"3753bde0144d808eb15c755b7176386c"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -373,7 +374,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-aco","id":"e4b1b5ebc172f2657485e41c35ad1cd7"},{"cmd":"packages/api-audit-logs --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-audit-logs","id":"b36aac5f0e34dc4583e5422ae589f1ed"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-elasticsearch","id":"b0f477d6b209f654714809b318be888e"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-elasticsearch-tasks","id":"580a9577fdbd4a241034a42e1a47dee5"},{"cmd":"packages/api-file-manager --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-file-manager","id":"346430a79981d3e214c87254a08e31b2"},{"cmd":"packages/api-form-builder --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-form-builder","id":"d386cddfd3c366ad9955193dcfe74363"},{"cmd":"packages/api-form-builder-so-ddb-es diff --git a/.github/workflows/pushNext.yml b/.github/workflows/pushNext.yml index 044aba6a3bd..a17491a6307 100644 --- a/.github/workflows/pushNext.yml +++ b/.github/workflows/pushNext.yml @@ -278,7 +278,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-aco","id":"8f23ec33f547aa62236f5c71115688d6"},{"cmd":"packages/api-audit-logs --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-audit-logs","id":"a292444cd9100f78d8fc196274393ea8"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-elasticsearch","id":"5963079c60b96202bbaf2a802ad14383"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-elasticsearch-tasks","id":"d81ad1d024a8746cc440e2e548770f8f"},{"cmd":"packages/api-file-manager --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-file-manager","id":"d6f293add4a252b96cbd770ab6e80557"},{"cmd":"packages/api-form-builder --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-form-builder","id":"3753bde0144d808eb15c755b7176386c"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -373,7 +374,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-aco","id":"e4b1b5ebc172f2657485e41c35ad1cd7"},{"cmd":"packages/api-audit-logs --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-audit-logs","id":"b36aac5f0e34dc4583e5422ae589f1ed"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-elasticsearch","id":"b0f477d6b209f654714809b318be888e"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-elasticsearch-tasks","id":"580a9577fdbd4a241034a42e1a47dee5"},{"cmd":"packages/api-file-manager --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-file-manager","id":"346430a79981d3e214c87254a08e31b2"},{"cmd":"packages/api-form-builder --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-form-builder","id":"d386cddfd3c366ad9955193dcfe74363"},{"cmd":"packages/api-form-builder-so-ddb-es diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index a1a6562f73e..60f304b84e4 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -73,7 +73,7 @@ const createJestTestsJobs = (storage: string | null) => { { name: "Packages to test with Jest", id: "list-packages", - run: "echo ${{ steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}" + run: "echo '${{ steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}'" } ] }); @@ -153,7 +153,9 @@ export const pullRequests = createWorkflow({ "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}", "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}", "is-fork-pr": "${{ steps.is-fork-pr.outputs.is-fork-pr }}", - "changed-packages": "${{ steps.detect-changed-packages.outputs.changed-packages }}" + "changed-packages": "${{ steps.detect-changed-packages.outputs.changed-packages }}", + "latest-webiny-version": + "${{ steps.latest-webiny-version.outputs.latest-webiny-version }}" }, steps: [ { @@ -197,6 +199,21 @@ export const pullRequests = createWorkflow({ "${{ steps.detect-changed-files.outputs.changed_files }}", { outputAs: "changed-packages" } ) + }, + { + name: "Get latest Webiny version on NPM", + id: "latest-webiny-version", + run: addToOutputs("latest-webiny-version", "$(npm view @webiny/cli version)") + } + ] + }), + assignMilestone: createJob({ + name: "Assign milestone", + needs: "constants", + steps: [ + { + name: "Print latest Webiny version", + run: "echo ${{ needs.constants.outputs.latest-webiny-version }}" } ] }), @@ -216,7 +233,7 @@ export const pullRequests = createWorkflow({ ] }), staticCodeAnalysis: createJob({ - needs: ["constants", "build"], + needs: ["constants"], name: "Static code analysis", checkout: { path: DIR_WEBINY_JS }, steps: [ diff --git a/.github/workflows/wac/utils/listPackagesWithJestTests.ts b/.github/workflows/wac/utils/listPackagesWithJestTests.ts index bc7f2e6f6a9..6168af7081b 100644 --- a/.github/workflows/wac/utils/listPackagesWithJestTests.ts +++ b/.github/workflows/wac/utils/listPackagesWithJestTests.ts @@ -266,8 +266,12 @@ const CUSTOM_HANDLERS: Record Array> = { "api-elasticsearch": () => { return [ { - cmd: "packages/api-elasticsearch", - storage: ["ddb-es", "ddb-os"] + cmd: "packages/api-elasticsearch --storage=ddb-es,ddb", + storage: ["ddb-es"] + }, + { + cmd: "packages/api-elasticsearch --storage=ddb-os,ddb", + storage: ["ddb-os"] } ]; }, diff --git a/package.json b/package.json index 6eccc7a300f..422a158c55e 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "@babel/code-frame": "^7.26.2", "@babel/compat-data": "^7.26.2", "@babel/core": "^7.26.0", - "@babel/helper-define-polyfill-provider": "^0.6.2", + "@babel/helper-define-polyfill-provider": "^0.6.3", "@babel/helper-environment-visitor": "^7.24.7", "@babel/parser": "^7.26.2", "@babel/plugin-proposal-class-properties": "^7.18.6", diff --git a/packages/api-dynamodb-to-elasticsearch/src/Operations.ts b/packages/api-dynamodb-to-elasticsearch/src/Operations.ts index d69ec29a1ed..0a52c7cb0e4 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/Operations.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/Operations.ts @@ -13,12 +13,20 @@ export enum OperationType { } export class Operations implements IOperations { - public readonly items: GenericRecord[] = []; + private _items: GenericRecord[] = []; + + public get items(): GenericRecord[] { + return this._items; + } public get total(): number { return this.items.length; } + public clear() { + this._items = []; + } + public insert(params: IInsertOperationParams): void { this.items.push( { diff --git a/packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts b/packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts new file mode 100644 index 00000000000..01eb1dfec10 --- /dev/null +++ b/packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts @@ -0,0 +1,72 @@ +import { + Context, + IDeleteOperationParams, + IInsertOperationParams, + IModifyOperationParams, + IOperations +} from "~/types"; +import { Operations } from "~/Operations"; +import { executeWithRetry, IExecuteWithRetryParams } from "~/executeWithRetry"; +import { ITimer } from "@webiny/handler-aws"; + +export type ISynchronizationBuilderExecuteWithRetryParams = Omit< + IExecuteWithRetryParams, + "context" | "timer" | "maxRunningTime" | "operations" +>; + +export interface ISynchronizationBuilder { + insert(params: IInsertOperationParams): void; + delete(params: IDeleteOperationParams): void; + build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise; +} + +export interface ISynchronizationBuilderParams { + timer: ITimer; + context: Pick; +} + +export class SynchronizationBuilder implements ISynchronizationBuilder { + private readonly timer: ITimer; + private readonly context: Pick; + private readonly operations: IOperations; + + public constructor(params: ISynchronizationBuilderParams) { + this.timer = params.timer; + this.context = params.context; + this.operations = new Operations(); + } + + public insert(params: IInsertOperationParams): void { + return this.operations.insert(params); + } + + public modify(params: IModifyOperationParams): void { + return this.operations.modify(params); + } + + public delete(params: IDeleteOperationParams): void { + return this.operations.delete(params); + } + + public build() { + return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => { + if (this.operations.total === 0) { + return; + } + await executeWithRetry({ + ...params, + maxRunningTime: this.timer.getRemainingMilliseconds(), + timer: this.timer, + context: this.context, + operations: this.operations + }); + this.operations.clear(); + }; + } +} + +export const createSynchronizationBuilder = ( + params: ISynchronizationBuilderParams +): ISynchronizationBuilder => { + return new SynchronizationBuilder(params); +}; diff --git a/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts b/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts index dcd6e4fb87f..c683a72c9a6 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts @@ -1,14 +1,8 @@ -import { getNumberEnvVariable } from "~/helpers/getNumberEnvVariable"; import { createDynamoDBEventHandler, timerFactory } from "@webiny/handler-aws"; +import { Context } from "~/types"; import { Decompressor } from "~/Decompressor"; import { OperationsBuilder } from "~/OperationsBuilder"; import { executeWithRetry } from "~/executeWithRetry"; -import { Context } from "~/types"; - -const MAX_PROCESSOR_PERCENT = getNumberEnvVariable( - "MAX_ES_PROCESSOR", - process.env.NODE_ENV === "test" ? 101 : 98 -); /** * Also, we need to set the maximum running time for the Lambda Function. @@ -49,7 +43,6 @@ export const createEventHandler = () => { await executeWithRetry({ timer, maxRunningTime: MAX_RUNNING_TIME, - maxProcessorPercent: MAX_PROCESSOR_PERCENT, context, operations }); diff --git a/packages/api-dynamodb-to-elasticsearch/src/execute.ts b/packages/api-dynamodb-to-elasticsearch/src/execute.ts index 44db8ce76b6..e98155ac7e4 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/execute.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/execute.ts @@ -30,8 +30,8 @@ export interface IExecuteParams { timer: ITimer; maxRunningTime: number; maxProcessorPercent: number; - context: Context; - operations: IOperations; + context: Pick; + operations: Pick; } const getError = (item: BulkOperationsResponseBodyItem): string | null => { @@ -67,6 +67,11 @@ const checkErrors = (result?: ApiResponse): void => export const execute = (params: IExecuteParams) => { return async (): Promise => { const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params; + + if (operations.total === 0) { + return; + } + const remainingTime = timer.getRemainingSeconds(); const runningTime = maxRunningTime - remainingTime; const maxWaitingTime = remainingTime - 90; diff --git a/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts b/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts index f9e442fe08d..2411b7a27a9 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts @@ -5,11 +5,17 @@ import { getNumberEnvVariable } from "./helpers/getNumberEnvVariable"; const minRemainingSecondsToTimeout = 120; -export interface IExecuteWithRetryParams extends IExecuteParams { +const MAX_PROCESSOR_PERCENT = getNumberEnvVariable( + "MAX_ES_PROCESSOR", + process.env.NODE_ENV === "test" ? 101 : 98 +); + +export interface IExecuteWithRetryParams extends Omit { maxRetryTime?: number; retries?: number; minTimeout?: number; maxTimeout?: number; + maxProcessorPercent?: number; } export const executeWithRetry = async (params: IExecuteWithRetryParams) => { @@ -35,7 +41,7 @@ export const executeWithRetry = async (params: IExecuteWithRetryParams) => { execute({ timer: params.timer, maxRunningTime: params.maxRunningTime, - maxProcessorPercent: params.maxProcessorPercent, + maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT, context: params.context, operations: params.operations }), diff --git a/packages/api-dynamodb-to-elasticsearch/src/index.ts b/packages/api-dynamodb-to-elasticsearch/src/index.ts index 99a39edec60..9c6d27a1d8b 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/index.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/index.ts @@ -6,4 +6,5 @@ export * from "./marshall"; export * from "./NotEnoughRemainingTimeError"; export * from "./Operations"; export * from "./OperationsBuilder"; +export * from "./SynchronizationBuilder"; export * from "./types"; diff --git a/packages/api-dynamodb-to-elasticsearch/src/types.ts b/packages/api-dynamodb-to-elasticsearch/src/types.ts index 142ba3f53b7..c01058c39eb 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/types.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/types.ts @@ -27,6 +27,7 @@ export interface IDeleteOperationParams { export interface IOperations { items: GenericRecord[]; total: number; + clear(): void; insert(params: IInsertOperationParams): void; modify(params: IModifyOperationParams): void; delete(params: IDeleteOperationParams): void; diff --git a/packages/api-elasticsearch-tasks/__tests__/helpers/useHandler.ts b/packages/api-elasticsearch-tasks/__tests__/helpers/useHandler.ts index 1de38dc6520..4eac9475d41 100644 --- a/packages/api-elasticsearch-tasks/__tests__/helpers/useHandler.ts +++ b/packages/api-elasticsearch-tasks/__tests__/helpers/useHandler.ts @@ -18,6 +18,7 @@ import { createElasticsearchBackgroundTasks } from "~/index"; import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; import dbPlugins from "@webiny/handler-db"; import { DynamoDbDriver } from "@webiny/db-dynamodb"; +import { createLogger } from "@webiny/api-log"; export interface UseHandlerParams { plugins?: PluginCollection; @@ -46,6 +47,9 @@ export const useHandler = (params?: UseHandlerParams) => { permissions: createPermissions(), identity: createIdentity() }), + createLogger({ + documentClient + }), i18nContext(), i18nStorage.storageOperations, createDummyLocales(), diff --git a/packages/api-elasticsearch-tasks/__tests__/mocks/context.ts b/packages/api-elasticsearch-tasks/__tests__/mocks/context.ts index 26deffab708..3c019b30085 100644 --- a/packages/api-elasticsearch-tasks/__tests__/mocks/context.ts +++ b/packages/api-elasticsearch-tasks/__tests__/mocks/context.ts @@ -8,11 +8,14 @@ import { IUpdateTaskResponse } from "@webiny/tasks/types"; import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; +// @ts-expect-error +import { createMockApiLog } from "@webiny/project-utils/testing/mockApiLog"; export const createContextMock = ( params?: PartialDeep ): Context & ElasticsearchContext => { return { + logger: createMockApiLog(), tenancy: { listTenants: async () => { return [ diff --git a/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts b/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts index 860790ed613..ad4aa8979bc 100644 --- a/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts +++ b/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts @@ -1,5 +1,5 @@ import { TaskManagerStore } from "@webiny/tasks/runner/TaskManagerStore"; -import { Context, ITask, ITaskLog } from "@webiny/tasks/types"; +import { Context, ITask, ITaskDataInput, ITaskLog } from "@webiny/tasks/types"; import { createTaskMock } from "~tests/mocks/task"; import { createContextMock } from "~tests/mocks/context"; import { createTaskLogMock } from "~tests/mocks/log"; @@ -9,11 +9,11 @@ interface Params { task?: ITask; log?: ITaskLog; } -export const createTaskManagerStoreMock = (params?: Params) => { +export const createTaskManagerStoreMock = (params?: Params) => { const context = params?.context || createContextMock(); const task = params?.task || createTaskMock(); const log = params?.log || createTaskLogMock(task); - return new TaskManagerStore({ + return new TaskManagerStore({ context, task, log diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts new file mode 100644 index 00000000000..ad54873fc86 --- /dev/null +++ b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts @@ -0,0 +1,172 @@ +import { ElasticsearchToDynamoDbSynchronization } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization"; +import { useHandler } from "~tests/helpers/useHandler"; +import { createManagers } from "./managers"; +import { ElasticsearchFetcher } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher"; +import { ElasticsearchSynchronize } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize"; +import { DATA_SYNCHRONIZATION_TASK } from "~/tasks"; +import { Context, SynchronizationBuilder } from "@webiny/api-dynamodb-to-elasticsearch"; +import { ITimer } from "@webiny/handler-aws"; +import { IIndexManager } from "~/settings/types"; + +const queryAllRecords = (index: string) => { + return { + index, + body: { + query: { + match_all: {} + }, + size: 10000, + _source: false + } + }; +}; + +interface ICreateSyncBuilderParams { + records: number; + timer: ITimer; + context: Pick; + index: string; +} + +const createRecordsFactory = (params: ICreateSyncBuilderParams) => { + const { timer, context, index, records } = params; + const syncBuilder = new SynchronizationBuilder({ + timer, + context + }); + + for (let i = 0; i < records; i++) { + syncBuilder.insert({ + id: `pkValue${i}:skValue${i}`, + index, + data: { + id: `skValue${i}`, + aText: `myText - ${i}` + } + }); + } + return { + run: () => { + return syncBuilder.build()(); + } + }; +}; + +const getTaskIndex = async (manager: IIndexManager): Promise => { + const indexes = await manager.list(); + const index = indexes.find( + index => index.includes("webinytask") && index.includes("-headless-cms-") + ); + if (!index) { + throw new Error("No index found."); + } + return index; +}; + +describe("ElasticsearchToDynamoDbSynchronization", () => { + it("should run a sync without any indexes and throw an error", async () => { + const handler = useHandler(); + + const context = await handler.rawHandle(); + + const { manager, indexManager } = createManagers({ + context + }); + + const sync = new ElasticsearchToDynamoDbSynchronization({ + manager, + indexManager, + fetcher: new ElasticsearchFetcher({ + client: context.elasticsearch + }), + synchronize: new ElasticsearchSynchronize({ + context, + timer: manager.timer + }) + }); + + try { + const result = await sync.run({ + flow: "elasticsearchToDynamoDb" + }); + expect(result).toEqual("Should not reach this point."); + } catch (ex) { + expect(ex.message).toBe("No Elasticsearch / OpenSearch indexes found."); + } + }); + + it("should run a sync with indexes and finish", async () => { + const handler = useHandler(); + + const context = await handler.rawHandle(); + + await context.tasks.createTask({ + definitionId: DATA_SYNCHRONIZATION_TASK, + input: { + flow: "elasticsearchToDynamoDb" + }, + name: "Data Sync Mock Task" + }); + + const { manager, indexManager } = createManagers({ + context + }); + + const index = await getTaskIndex(indexManager); + + const totalMockItemsToInsert = 101; + const recordsFactory = createRecordsFactory({ + context, + index, + timer: manager.timer, + records: totalMockItemsToInsert + }); + try { + await recordsFactory.run(); + } catch (ex) { + expect(ex.message).toBe("Should not reach this point."); + } + /** + * Now we need to make sure that the mock data is in the index. + */ + const response = await context.elasticsearch.search(queryAllRecords(index)); + expect(response.body.hits.hits).toHaveLength(totalMockItemsToInsert + 1); + + const sync = new ElasticsearchToDynamoDbSynchronization({ + manager, + indexManager, + fetcher: new ElasticsearchFetcher({ + client: context.elasticsearch + }), + synchronize: new ElasticsearchSynchronize({ + context, + timer: manager.timer + }) + }); + + const result = await sync.run({ + flow: "elasticsearchToDynamoDb" + }); + expect(result).toEqual({ + delay: -1, + input: { + elasticsearchToDynamoDb: { + finished: true + }, + flow: "elasticsearchToDynamoDb" + }, + locale: "en-US", + message: undefined, + status: "continue", + tenant: "root", + wait: undefined, + webinyTaskDefinitionId: "mockDefinitionId", + webinyTaskId: "mockEventId" + }); + /** + * Now we need to make sure that the mock data is not in the index anymore. + */ + const afterRunResponse = await context.elasticsearch.search(queryAllRecords(index)); + expect(afterRunResponse.body.hits.hits).toHaveLength(1); + }); +}); diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts new file mode 100644 index 00000000000..7f2769d1edc --- /dev/null +++ b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts @@ -0,0 +1,119 @@ +import { createDataSynchronization, DATA_SYNCHRONIZATION_TASK } from "~/tasks"; +import { TaskDefinitionPlugin, TaskResponseStatus } from "@webiny/tasks"; +import { createRunner } from "@webiny/project-utils/testing/tasks"; +import { useHandler } from "~tests/helpers/useHandler"; +import { IDataSynchronizationInput, IFactories } from "~/tasks/dataSynchronization/types"; + +jest.mock("~/tasks/dataSynchronization/createFactories", () => { + return { + createFactories: (): IFactories => { + return { + elasticsearchToDynamoDb: ({ manager }) => { + return { + run: async input => { + return manager.response.continue({ + ...input, + elasticsearchToDynamoDb: { + finished: true + } + }); + } + }; + } + }; + } + }; +}); + +describe("data synchronization - elasticsearch", () => { + it("should create a task definition", async () => { + const result = createDataSynchronization(); + + expect(result).toBeInstanceOf(TaskDefinitionPlugin); + expect(result).toEqual({ + isPrivate: false, + task: { + id: DATA_SYNCHRONIZATION_TASK, + isPrivate: false, + title: "Data Synchronization", + description: "Synchronize data between Elasticsearch and DynamoDB", + maxIterations: 100, + disableDatabaseLogs: true, + fields: [], + run: expect.any(Function), + createInputValidation: expect.any(Function) + } + }); + }); + + it("should run a task and end with error due to invalid flow", async () => { + const handler = useHandler({}); + + const context = await handler.rawHandle(); + + try { + const task = await context.tasks.createTask({ + definitionId: DATA_SYNCHRONIZATION_TASK, + input: { + // @ts-expect-error + flow: "unknownFlow" + }, + name: "Data Sync Mock Task" + }); + expect(task).toEqual("Should not reach this point."); + } catch (ex) { + expect(ex.message).toEqual("Validation failed."); + expect(ex.data).toEqual({ + invalidFields: { + flow: { + code: "invalid_enum_value", + data: { + fatal: undefined, + path: ["flow"] + }, + message: + "Invalid enum value. Expected 'elasticsearchToDynamoDb', received 'unknownFlow'" + } + } + }); + } + }); + + it("should run a task and end with done", async () => { + const handler = useHandler({}); + + const context = await handler.rawHandle(); + + const task = await context.tasks.createTask({ + definitionId: DATA_SYNCHRONIZATION_TASK, + input: { + flow: "elasticsearchToDynamoDb" + }, + name: "Data Sync Mock Task" + }); + + const runner = createRunner({ + context, + task: createDataSynchronization(), + onContinue: async () => { + return; + } + }); + + const result = await runner({ + webinyTaskId: task.id + }); + + expect(result).toEqual({ + status: TaskResponseStatus.DONE, + webinyTaskId: task.id, + webinyTaskDefinitionId: DATA_SYNCHRONIZATION_TASK, + tenant: "root", + locale: "en-US", + message: undefined, + output: undefined + }); + const taskCheck = await context.tasks.getTask(task.id); + expect(taskCheck?.iterations).toEqual(2); + }); +}); diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts new file mode 100644 index 00000000000..1ac6803de3c --- /dev/null +++ b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts @@ -0,0 +1,40 @@ +import { IndexManager } from "~/settings"; +import { + IDataSynchronizationInput, + IDataSynchronizationManager +} from "~/tasks/dataSynchronization/types"; +import { Context } from "~/types"; +import { Manager } from "~/tasks/Manager"; +import { Response, TaskResponse } from "@webiny/tasks"; +import { createMockEvent } from "~tests/mocks/event"; +import { createTaskManagerStoreMock } from "~tests/mocks/store"; +import { timerFactory } from "@webiny/handler-aws/utils"; + +export interface ICreateManagersParams { + context: Context; +} + +export const createManagers = (params: ICreateManagersParams) => { + const { context } = params; + const manager = new Manager({ + elasticsearchClient: context.elasticsearch, + // @ts-expect-error + documentClient: context.db.driver.documentClient, + response: new TaskResponse(new Response(createMockEvent())), + context, + isAborted: () => { + return false; + }, + isCloseToTimeout: () => { + return false; + }, + timer: timerFactory(), + store: createTaskManagerStoreMock() + }); + + const indexManager = new IndexManager(context.elasticsearch, {}); + return { + manager: manager as unknown as IDataSynchronizationManager, + indexManager + }; +}; diff --git a/packages/api-elasticsearch-tasks/jest.setup.js b/packages/api-elasticsearch-tasks/jest.setup.js index 049095053ae..e9fae563a72 100644 --- a/packages/api-elasticsearch-tasks/jest.setup.js +++ b/packages/api-elasticsearch-tasks/jest.setup.js @@ -1,6 +1,8 @@ const base = require("../../jest.config.base"); const presets = require("@webiny/project-utils/testing/presets")( ["@webiny/api-headless-cms", "storage-operations"], + ["@webiny/api-form-builder", "storage-operations"], + ["@webiny/api-page-builder", "storage-operations"], ["@webiny/api-i18n", "storage-operations"], ["@webiny/api-security", "storage-operations"], ["@webiny/api-tenancy", "storage-operations"] diff --git a/packages/api-elasticsearch-tasks/package.json b/packages/api-elasticsearch-tasks/package.json index 48c0357b022..45f31a3ce85 100644 --- a/packages/api-elasticsearch-tasks/package.json +++ b/packages/api-elasticsearch-tasks/package.json @@ -13,8 +13,11 @@ "license": "MIT", "dependencies": { "@webiny/api": "0.0.0", + "@webiny/api-dynamodb-to-elasticsearch": "0.0.0", "@webiny/api-elasticsearch": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/aws-sdk": "0.0.0", + "@webiny/db": "0.0.0", "@webiny/db-dynamodb": "0.0.0", "@webiny/error": "0.0.0", "@webiny/tasks": "0.0.0", @@ -47,5 +50,12 @@ "build": "yarn webiny run build", "watch": "yarn webiny run watch" }, - "gitHead": "8476da73b653c89cc1474d968baf55c1b0ae0e5f" + "gitHead": "8476da73b653c89cc1474d968baf55c1b0ae0e5f", + "adio": { + "ignore": { + "src": [ + "node:util" + ] + } + } } diff --git a/packages/api-elasticsearch-tasks/src/definitions/entry.ts b/packages/api-elasticsearch-tasks/src/definitions/entry.ts index b985c523455..fa3c9bb7afd 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/entry.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/entry.ts @@ -1,7 +1,7 @@ -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox"; interface Params { - table: Table; + table: TableDef; entityName: string; } diff --git a/packages/api-elasticsearch-tasks/src/definitions/table.ts b/packages/api-elasticsearch-tasks/src/definitions/table.ts index cc1f5e7a50e..5bd160dfb1e 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/table.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/table.ts @@ -1,11 +1,11 @@ import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { Table, TableConstructor } from "@webiny/db-dynamodb/toolbox"; +import { Table, TableConstructor, TableDef } from "@webiny/db-dynamodb/toolbox"; interface Params { documentClient: DynamoDBDocument; } -export const createTable = ({ documentClient }: Params): Table => { +export const createTable = ({ documentClient }: Params): TableDef => { const config: TableConstructor = { name: process.env.DB_TABLE_ELASTICSEARCH as string, partitionKey: "PK", diff --git a/packages/api-elasticsearch-tasks/src/helpers/scan.ts b/packages/api-elasticsearch-tasks/src/helpers/scan.ts index 1f5d4141dbf..2440c0deba5 100644 --- a/packages/api-elasticsearch-tasks/src/helpers/scan.ts +++ b/packages/api-elasticsearch-tasks/src/helpers/scan.ts @@ -1,11 +1,11 @@ import { scan as tableScan, ScanOptions } from "@webiny/db-dynamodb"; -import { Table } from "@webiny/db-dynamodb/toolbox"; +import { TableDef } from "@webiny/db-dynamodb/toolbox"; import { IElasticsearchIndexingTaskValuesKeys } from "~/types"; interface Params { - table: Table; + table: TableDef; keys?: IElasticsearchIndexingTaskValuesKeys; - options?: Pick; + options?: ScanOptions; } export const scan = async (params: Params) => { @@ -13,9 +13,9 @@ export const scan = async (params: Params) => { return tableScan({ table, options: { + ...params.options, startKey: keys, - limit: 200, - ...params.options + limit: params.options?.limit || 200 } }); }; diff --git a/packages/api-elasticsearch-tasks/src/index.ts b/packages/api-elasticsearch-tasks/src/index.ts index e8ca3b86175..12dfab884d1 100644 --- a/packages/api-elasticsearch-tasks/src/index.ts +++ b/packages/api-elasticsearch-tasks/src/index.ts @@ -1,4 +1,5 @@ import { + createDataSynchronization, createElasticsearchReindexingTask, createEnableIndexingTask, createIndexesTaskDefinition @@ -14,7 +15,8 @@ export const createElasticsearchBackgroundTasks = ( return [ createElasticsearchReindexingTask(params), createEnableIndexingTask(params), - createIndexesTaskDefinition(params) + createIndexesTaskDefinition(params), + createDataSynchronization(params) ]; }; diff --git a/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts b/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts index 289e70cd2b9..94a9dca0d32 100644 --- a/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts +++ b/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts @@ -11,6 +11,22 @@ const defaultIndexSettings: IIndexSettingsValues = { refreshInterval: "1s" }; +export interface IListIndicesResponse { + index: string; +} + +const indexPrefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || ""; +const filterIndex = (item?: string) => { + if (!item) { + return false; + } else if (item.startsWith(".")) { + return false; + } else if (indexPrefix) { + return item.startsWith(indexPrefix); + } + return true; +}; + export class IndexManager implements IIndexManager { private readonly client: Client; private readonly disable: DisableIndexing; @@ -41,13 +57,13 @@ export class IndexManager implements IIndexManager { public async list(): Promise { try { - const response = await this.client.cat.indices({ + const response = await this.client.cat.indices({ format: "json" }); if (!Array.isArray(response.body)) { return []; } - return response.body.map((item: any) => item.index).filter(Boolean); + return response.body.map(item => item.index).filter(filterIndex); } catch (ex) { console.error( JSON.stringify({ diff --git a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts index 1fbebc97913..a62664742a0 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts @@ -1,11 +1,11 @@ import { DynamoDBDocument, getDocumentClient } from "@webiny/aws-sdk/client-dynamodb"; import { Client, createElasticsearchClient } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; -import { Context, IElasticsearchIndexingTaskValues, IManager } from "~/types"; +import { Context, IManager } from "~/types"; import { createEntry } from "~/definitions/entry"; import { Entity } from "@webiny/db-dynamodb/toolbox"; import { ITaskResponse } from "@webiny/tasks/response/abstractions"; -import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; +import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; import { batchReadAll, BatchReadItem, @@ -13,30 +13,33 @@ import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb"; +import { ITimer } from "@webiny/handler-aws/utils"; -export interface ManagerParams { +export interface ManagerParams { context: Context; documentClient?: DynamoDBDocument; elasticsearchClient?: Client; - isCloseToTimeout: () => boolean; + isCloseToTimeout: IIsCloseToTimeoutCallable; isAborted: () => boolean; response: ITaskResponse; - store: ITaskManagerStore; + store: ITaskManagerStore; + timer: ITimer; } -export class Manager implements IManager { +export class Manager implements IManager { public readonly documentClient: DynamoDBDocument; public readonly elasticsearch: Client; public readonly context: Context; public readonly table: ReturnType; - public readonly isCloseToTimeout: () => boolean; + public readonly isCloseToTimeout: IIsCloseToTimeoutCallable; public readonly isAborted: () => boolean; public readonly response: ITaskResponse; - public readonly store: ITaskManagerStore; + public readonly store: ITaskManagerStore; + public readonly timer: ITimer; private readonly entities: Record> = {}; - public constructor(params: ManagerParams) { + public constructor(params: ManagerParams) { this.context = params.context; this.documentClient = params?.documentClient || getDocumentClient(); @@ -58,6 +61,7 @@ export class Manager implements IManager { }; this.response = params.response; this.store = params.store; + this.timer = params.timer; } public getEntity(name: string): Entity { diff --git a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts index c9d65ce92e4..62b572e0845 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts @@ -6,12 +6,16 @@ import { CreateElasticsearchIndexTaskPluginIndex } from "./CreateElasticsearchIndexTaskPlugin"; import { Context } from "~/types"; +import { IElasticsearchCreateIndexesTaskInput } from "~/tasks/createIndexes/types"; export class CreateIndexesTaskRunner { - private readonly manager: Manager; + private readonly manager: Manager; private readonly indexManager: IndexManager; - public constructor(manager: Manager, indexManager: IndexManager) { + public constructor( + manager: Manager, + indexManager: IndexManager + ) { this.manager = manager; this.indexManager = indexManager; diff --git a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts index a84529c4b74..46850d171b2 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts @@ -12,23 +12,24 @@ export const createIndexesTaskDefinition = (params?: IElasticsearchTaskConfig) = * No point in having more than 2 runs, as the create index operations should not even take 1 full run, no matter how much indeexs is there to create. */ maxIterations: 2, - run: async ({ response, context, isCloseToTimeout, isAborted, store, input }) => { + run: async ({ response, context, isCloseToTimeout, isAborted, store, input, timer }) => { const { Manager } = await import( - /* webpackChunkName: "ElasticsearchTaskManager" */ + /* webpackChunkName: "Manager" */ "../Manager" ); const { IndexManager } = await import( - /* webpackChunkName: "ElasticsearchTaskSettings" */ "~/settings" + /* webpackChunkName: "IndexManager" */ "~/settings" ); - const manager = new Manager({ + const manager = new Manager({ elasticsearchClient: params?.elasticsearchClient, documentClient: params?.documentClient, response, context, isAborted, isCloseToTimeout, - store + store, + timer }); const indexManager = new IndexManager(manager.elasticsearch, {}); diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts new file mode 100644 index 00000000000..5e3b1231eae --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts @@ -0,0 +1,69 @@ +import { + IDataSynchronizationInput, + IDataSynchronizationManager, + IFactories +} from "~/tasks/dataSynchronization/types"; +import { IIndexManager } from "~/settings/types"; +import { ElasticsearchSynchronize } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize"; +import { ElasticsearchFetcher } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher"; + +export interface IDataSynchronizationTaskRunnerParams { + manager: IDataSynchronizationManager; + indexManager: IIndexManager; + factories: IFactories; +} + +export class DataSynchronizationTaskRunner { + private readonly manager: IDataSynchronizationManager; + private readonly indexManager: IIndexManager; + private readonly factories: IFactories; + + public constructor(params: IDataSynchronizationTaskRunnerParams) { + this.manager = params.manager; + this.indexManager = params.indexManager; + this.factories = params.factories; + } + + public async run(input: IDataSynchronizationInput) { + this.validateFlow(input); + /** + * Go through the Elasticsearch and delete records which do not exist in the Elasticsearch table. + */ + // + if (input.flow === "elasticsearchToDynamoDb" && !input.elasticsearchToDynamoDb?.finished) { + const sync = this.factories.elasticsearchToDynamoDb({ + manager: this.manager, + indexManager: this.indexManager, + synchronize: new ElasticsearchSynchronize({ + context: this.manager.context, + timer: this.manager.timer + }), + fetcher: new ElasticsearchFetcher({ + client: this.manager.elasticsearch + }) + }); + try { + return await sync.run(input); + } catch (ex) { + return this.manager.response.error(ex); + } + } + /** + * We are done. + */ + return this.manager.response.done(); + } + + private validateFlow(input: IDataSynchronizationInput): void { + if (!input.flow) { + throw new Error(`Missing "flow" in the input.`); + } else if (this.factories[input.flow]) { + return; + } + throw new Error( + `Invalid flow "${input.flow}". Allowed flows: ${Object.keys(this.factories).join( + ", " + )}.` + ); + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts new file mode 100644 index 00000000000..dd1e20cecb3 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts @@ -0,0 +1,10 @@ +import { IFactories } from "./types"; +import { ElasticsearchToDynamoDbSynchronization } from "./elasticsearch/ElasticsearchToDynamoDbSynchronization"; + +export const createFactories = (): IFactories => { + return { + elasticsearchToDynamoDb: params => { + return new ElasticsearchToDynamoDbSynchronization(params); + } + }; +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts new file mode 100644 index 00000000000..55f58abf601 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts @@ -0,0 +1,107 @@ +import { Client } from "@webiny/api-elasticsearch"; +import { + IElasticsearchFetcher, + IElasticsearchFetcherFetchParams, + IElasticsearchFetcherFetchResponse, + IElasticsearchFetcherFetchResponseItem +} from "./abstractions/ElasticsearchFetcher"; +import { ElasticsearchSearchResponse, PrimitiveValue } from "@webiny/api-elasticsearch/types"; +import { shouldIgnoreEsResponseError } from "./shouldIgnoreEsResponseError"; +import { inspect } from "node:util"; + +export interface IElasticsearchFetcherParams { + client: Client; +} + +export class ElasticsearchFetcher implements IElasticsearchFetcher { + private readonly client: Client; + + public constructor(params: IElasticsearchFetcherParams) { + this.client = params.client; + } + public async fetch({ + index, + cursor, + limit + }: IElasticsearchFetcherFetchParams): Promise { + let response: ElasticsearchSearchResponse; + try { + response = await this.client.search({ + index, + body: { + query: { + match_all: {} + }, + sort: { + "id.keyword": { + order: "asc" + } + }, + size: limit + 1, + track_total_hits: true, + search_after: cursor, + _source: false + } + }); + } catch (ex) { + /** + * If we ignore the error, we can continue with the next index. + */ + if (shouldIgnoreEsResponseError(ex)) { + if (process.env.DEBUG === "true") { + console.error( + inspect(ex, { + depth: 5, + showHidden: true + }) + ); + } + return { + done: true, + totalCount: 0, + items: [] + }; + } + console.error("Failed to fetch data from Elasticsearch.", ex); + throw ex; + } + + const { hits, total } = response.body.hits; + if (hits.length === 0) { + return { + done: true, + cursor: undefined, + totalCount: total.value, + items: [] + }; + } + + const hasMoreItems = hits.length > limit; + let nextCursor: PrimitiveValue[] | undefined; + if (hasMoreItems) { + hits.pop(); + nextCursor = hits.at(-1)?.sort; + } + const items = hits.reduce((collection, hit) => { + const [PK, SK] = hit._id.split(":"); + if (!PK || !SK) { + return collection; + } + collection.push({ + PK, + SK, + _id: hit._id, + index: hit._index + }); + + return collection; + }, []); + + return { + totalCount: total.value, + cursor: nextCursor, + done: !nextCursor, + items + }; + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts new file mode 100644 index 00000000000..428109e460a --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts @@ -0,0 +1,101 @@ +import { batchReadAll } from "@webiny/db-dynamodb"; +import { createSynchronizationBuilder } from "@webiny/api-dynamodb-to-elasticsearch"; +import { + getElasticsearchEntity, + getElasticsearchEntityType, + getTable, + IGetElasticsearchEntityTypeParams +} from "~/tasks/dataSynchronization/entities"; +import { ITimer } from "@webiny/handler-aws"; +import { Context } from "~/types"; +import { + IElasticsearchSynchronize, + IElasticsearchSynchronizeExecuteParams, + IElasticsearchSynchronizeExecuteResponse +} from "./abstractions/ElasticsearchSynchronize"; + +export interface IElasticsearchSynchronizeParams { + timer: ITimer; + context: Context; +} + +interface IDynamoDbItem { + PK: string; + SK: string; +} + +export class ElasticsearchSynchronize implements IElasticsearchSynchronize { + private readonly timer: ITimer; + private readonly context: Context; + + public constructor(params: IElasticsearchSynchronizeParams) { + this.timer = params.timer; + this.context = params.context; + } + + public async execute( + params: IElasticsearchSynchronizeExecuteParams + ): Promise { + const { items, done, index } = params; + if (items.length === 0) { + return { + done: true + }; + } + + const table = getTable({ + type: "es", + context: this.context + }); + + const readableItems = items.map(item => { + const entity = this.getEntity(item); + return entity.item.getBatch({ + PK: item.PK, + SK: item.SK + }); + }); + + const tableItems = await batchReadAll({ + items: readableItems, + table + }); + + const elasticsearchSyncBuilder = createSynchronizationBuilder({ + timer: this.timer, + context: this.context + }); + /** + * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table. + */ + for (const item of items) { + const exists = tableItems.some(ddbItem => { + return ddbItem.PK === item.PK && ddbItem.SK === item.SK; + }); + if (exists) { + continue; + } + elasticsearchSyncBuilder.delete({ + index, + id: item._id + }); + } + + const executeWithRetry = elasticsearchSyncBuilder.build(); + await executeWithRetry(); + + return { + done + }; + } + + private getEntity( + params: IGetElasticsearchEntityTypeParams + ): ReturnType { + const type = getElasticsearchEntityType(params); + return getElasticsearchEntity({ + type, + context: this.context + }); + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts new file mode 100644 index 00000000000..78815e12fe7 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts @@ -0,0 +1,95 @@ +import { + IDataSynchronizationInput, + IDataSynchronizationManager, + IElasticsearchSyncParams, + ISynchronization, + ISynchronizationRunResult +} from "../types"; +import { IIndexManager } from "~/settings/types"; +import { NonEmptyArray } from "@webiny/api/types"; +import { IElasticsearchSynchronize } from "./abstractions/ElasticsearchSynchronize"; +import { IElasticsearchFetcher } from "./abstractions/ElasticsearchFetcher"; + +export class ElasticsearchToDynamoDbSynchronization implements ISynchronization { + private readonly manager: IDataSynchronizationManager; + private readonly indexManager: IIndexManager; + private readonly synchronize: IElasticsearchSynchronize; + private readonly fetcher: IElasticsearchFetcher; + + public constructor(params: IElasticsearchSyncParams) { + this.manager = params.manager; + this.indexManager = params.indexManager; + this.synchronize = params.synchronize; + this.fetcher = params.fetcher; + } + + public async run(input: IDataSynchronizationInput): Promise { + const lastIndex = input.elasticsearchToDynamoDb?.index; + let cursor = input.elasticsearchToDynamoDb?.cursor; + const indexes = await this.fetchAllIndexes(); + + let next = 0; + if (lastIndex) { + next = indexes.findIndex(index => index === lastIndex); + } + + let currentIndex = indexes[next]; + + while (currentIndex) { + if (this.manager.isAborted()) { + return this.manager.response.aborted(); + } + /** + * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly. + * We want to leave enough time for possible retries. + */ + // + else if (this.manager.isCloseToTimeout(180)) { + return this.manager.response.continue({ + ...input, + elasticsearchToDynamoDb: { + ...input.elasticsearchToDynamoDb, + index: currentIndex, + cursor + } + }); + } + + const result = await this.fetcher.fetch({ + index: currentIndex, + cursor, + limit: 100 + }); + + const syncResult = await this.synchronize.execute({ + done: result.done, + index: currentIndex, + items: result.items + }); + + if (!syncResult.done && result.cursor) { + cursor = result.cursor; + continue; + } + cursor = undefined; + + const next = indexes.findIndex(index => index === currentIndex) + 1; + currentIndex = indexes[next]; + } + + return this.manager.response.continue({ + ...input, + elasticsearchToDynamoDb: { + finished: true + } + }); + } + + private async fetchAllIndexes(): Promise> { + const result = await this.indexManager.list(); + if (result.length > 0) { + return result as NonEmptyArray; + } + throw new Error("No Elasticsearch / OpenSearch indexes found."); + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts new file mode 100644 index 00000000000..242bec7f4d8 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts @@ -0,0 +1,25 @@ +import { PrimitiveValue } from "@webiny/api-elasticsearch/types"; + +export interface IElasticsearchFetcherFetchResponseItem { + PK: string; + SK: string; + _id: string; + index: string; +} + +export interface IElasticsearchFetcherFetchParams { + index: string; + cursor?: PrimitiveValue[]; + limit: number; +} + +export interface IElasticsearchFetcherFetchResponse { + done: boolean; + totalCount: number; + cursor?: PrimitiveValue[]; + items: IElasticsearchFetcherFetchResponseItem[]; +} + +export interface IElasticsearchFetcher { + fetch(params: IElasticsearchFetcherFetchParams): Promise; +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts new file mode 100644 index 00000000000..09f48a5c763 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts @@ -0,0 +1,21 @@ +export interface IElasticsearchSynchronizeExecuteParamsItem { + PK: string; + SK: string; + _id: string; + index: string; +} + +export interface IElasticsearchSynchronizeExecuteParams { + done: boolean; + index: string; + items: IElasticsearchSynchronizeExecuteParamsItem[]; +} + +export interface IElasticsearchSynchronizeExecuteResponse { + done: boolean; +} +export interface IElasticsearchSynchronize { + execute( + params: IElasticsearchSynchronizeExecuteParams + ): Promise; +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts new file mode 100644 index 00000000000..b4d76ba15ad --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts @@ -0,0 +1,11 @@ +import WebinyError from "@webiny/error"; + +const IGNORED_ES_SEARCH_EXCEPTIONS = [ + "index_not_found_exception", + "search_phase_execution_exception", + "illegal_argument_exception" +]; + +export const shouldIgnoreEsResponseError = (error: WebinyError) => { + return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts new file mode 100644 index 00000000000..cc20f4061ef --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts @@ -0,0 +1,52 @@ +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { NonEmptyArray } from "@webiny/api/types"; +import { IRegistryItem } from "@webiny/db"; +import { EntityType } from "./getElasticsearchEntityType"; +import { Context } from "~/types"; + +export interface IGetElasticsearchEntityParams { + type: EntityType | unknown; + context: Pick; +} + +const createPredicate = (app: string, tags: NonEmptyArray) => { + return (item: IRegistryItem) => { + return item.app === app && tags.every(tag => item.tags.includes(tag)); + }; +}; + +export const getElasticsearchEntity = (params: IGetElasticsearchEntityParams) => { + const { type, context } = params; + + const getByPredicate = (predicate: (item: IRegistryItem) => boolean) => { + return context.db.registry.getOneItem(predicate); + }; + + try { + switch (type) { + case EntityType.CMS: + return getByPredicate(createPredicate("cms", ["es"])); + case EntityType.PAGE_BUILDER: + return getByPredicate(createPredicate("pb", ["es"])); + case EntityType.FORM_BUILDER: + return getByPredicate(createPredicate("fb", ["es"])); + case EntityType.FORM_BUILDER_SUBMISSION: + return getByPredicate(createPredicate("fb", ["es", "form-submission"])); + } + } catch (ex) {} + throw new Error(`Unknown entity type "${type}".`); +}; + +export interface IListElasticsearchEntitiesParams { + context: Pick; +} + +export const listElasticsearchEntities = ( + params: IListElasticsearchEntitiesParams +): IRegistryItem[] => { + const { context } = params; + + return context.db.registry.getItems(item => { + return item.tags.includes("es"); + }); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts new file mode 100644 index 00000000000..5501193cf75 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts @@ -0,0 +1,27 @@ +export enum EntityType { + CMS = "headless-cms", + PAGE_BUILDER = "page-builder", + FORM_BUILDER = "form-builder", + FORM_BUILDER_SUBMISSION = "form-builder-submission" +} + +export interface IGetElasticsearchEntityTypeParams { + SK: string; + index: string; +} + +export const getElasticsearchEntityType = ( + params: IGetElasticsearchEntityTypeParams +): EntityType => { + if (params.index.includes("-headless-cms-")) { + return EntityType.CMS; + } else if (params.index.endsWith("-page-builder")) { + return EntityType.PAGE_BUILDER; + } else if (params.index.endsWith("-form-builder")) { + if (params.SK.startsWith("FS#")) { + return EntityType.FORM_BUILDER_SUBMISSION; + } + return EntityType.FORM_BUILDER; + } + throw new Error(`Unknown entity type for item "${JSON.stringify(params)}".`); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts new file mode 100644 index 00000000000..bb0e5720b9b --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts @@ -0,0 +1,30 @@ +import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox"; +import { Context } from "~/types"; +import { NonEmptyArray } from "@webiny/api/types"; +import { IRegistryItem } from "@webiny/db"; + +export interface IGetTableParams { + context: Pick; + type: "regular" | "es"; +} + +const createPredicate = (app: string, tags: NonEmptyArray) => { + return (item: IRegistryItem) => { + return item.app === app && tags.every(tag => item.tags.includes(tag)); + }; +}; + +export const getTable = (params: IGetTableParams): TableDef => { + const { context, type } = params; + + const getByPredicate = (predicate: (item: IRegistryItem) => boolean) => { + const item = context.db.registry.getOneItem(predicate); + return item.item; + }; + + const entity = getByPredicate(createPredicate("cms", [type])); + if (!entity) { + throw new Error(`Unknown entity type "${type}".`); + } + return entity.table as TableDef; +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts new file mode 100644 index 00000000000..938b45e64ab --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts @@ -0,0 +1,3 @@ +export * from "./getElasticsearchEntity"; +export * from "./getElasticsearchEntityType"; +export * from "./getTable"; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts new file mode 100644 index 00000000000..19c4473f69b --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts @@ -0,0 +1,79 @@ +import { createTaskDefinition } from "@webiny/tasks"; +import { Context, IElasticsearchTaskConfig } from "~/types"; +import { + IDataSynchronizationInput, + IDataSynchronizationManager, + IDataSynchronizationOutput +} from "~/tasks/dataSynchronization/types"; + +export const DATA_SYNCHRONIZATION_TASK = "dataSynchronization"; + +export const createDataSynchronization = (params?: IElasticsearchTaskConfig) => { + return createTaskDefinition({ + id: DATA_SYNCHRONIZATION_TASK, + isPrivate: false, + title: "Data Synchronization", + description: "Synchronize data between Elasticsearch and DynamoDB", + maxIterations: 100, + disableDatabaseLogs: true, + async run({ context, response, isCloseToTimeout, isAborted, store, input, timer }) { + const { Manager } = await import( + /* webpackChunkName: "Manager" */ + "../Manager" + ); + + const { IndexManager } = await import( + /* webpackChunkName: "IndexManager" */ "~/settings" + ); + + const manager = new Manager({ + elasticsearchClient: params?.elasticsearchClient, + documentClient: params?.documentClient, + response, + context, + isAborted, + isCloseToTimeout, + store, + timer + }); + + const indexManager = new IndexManager(manager.elasticsearch, {}); + + const { DataSynchronizationTaskRunner } = await import( + /* webpackChunkName: "DataSynchronizationTaskRunner" */ "./DataSynchronizationTaskRunner" + ); + + const { createFactories } = await import( + /* webpackChunkName: "createFactories" */ "./createFactories" + ); + + try { + const dataSynchronization = new DataSynchronizationTaskRunner({ + manager: manager as unknown as IDataSynchronizationManager, + indexManager, + factories: createFactories() + }); + return await dataSynchronization.run({ + ...input + }); + } catch (ex) { + return response.error(ex); + } + }, + createInputValidation({ validator }) { + return { + flow: validator.enum(["elasticsearchToDynamoDb"]), + elasticsearchToDynamoDb: validator + .object({ + finished: validator.boolean().optional().default(false), + index: validator.string().optional(), + cursor: validator.array(validator.string()).optional() + }) + .optional() + .default({ + finished: false + }) + }; + } + }); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts new file mode 100644 index 00000000000..c73f7f2d573 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts @@ -0,0 +1,62 @@ +import { IManager } from "~/types"; +import { PrimitiveValue } from "@webiny/api-elasticsearch/types"; +import { IIndexManager } from "~/settings/types"; +import { + ITaskResponseAbortedResult, + ITaskResponseContinueResult, + ITaskResponseDoneResult, + ITaskResponseDoneResultOutput, + ITaskResponseErrorResult +} from "@webiny/tasks"; +import { IElasticsearchSynchronize } from "~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize"; +import { IElasticsearchFetcher } from "~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher"; + +export interface IDataSynchronizationInputValue { + finished?: boolean; +} + +export interface IDataSynchronizationInputElasticsearchToDynamoDbValue + extends IDataSynchronizationInputValue { + index?: string; + cursor?: PrimitiveValue[]; +} + +export interface IDataSynchronizationInput { + flow: "elasticsearchToDynamoDb"; + elasticsearchToDynamoDb?: IDataSynchronizationInputElasticsearchToDynamoDbValue; +} + +export type IDataSynchronizationOutput = ITaskResponseDoneResultOutput; + +export type ISynchronizationRunResult = + | ITaskResponseContinueResult + | ITaskResponseDoneResult + | ITaskResponseErrorResult + | ITaskResponseAbortedResult; + +export interface ISynchronization { + run(input: IDataSynchronizationInput): Promise; +} + +export interface IElasticsearchSyncParams { + manager: IDataSynchronizationManager; + indexManager: IIndexManager; + synchronize: IElasticsearchSynchronize; + fetcher: IElasticsearchFetcher; +} + +export interface IElasticsearchSyncFactory { + (params: IElasticsearchSyncParams): ISynchronization; +} + +export interface IFactories { + /** + * Delete all the records which are in the Elasticsearch but not in the Elasticsearch DynamoDB table. + */ + elasticsearchToDynamoDb: IElasticsearchSyncFactory; +} + +export type IDataSynchronizationManager = IManager< + IDataSynchronizationInput, + IDataSynchronizationOutput +>; diff --git a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts index 938477cbbbf..3dca288e2d4 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts @@ -2,13 +2,17 @@ import { IManager } from "~/types"; import { ITaskResponse, ITaskResponseResult } from "@webiny/tasks/response/abstractions"; import { IndexManager } from "~/settings"; import { IIndexManager } from "~/settings/types"; +import { IElasticsearchEnableIndexingTaskInput } from "~/tasks/enableIndexing/types"; export class EnableIndexingTaskRunner { - private readonly manager: IManager; + private readonly manager: IManager; private readonly indexManager: IIndexManager; private readonly response: ITaskResponse; - public constructor(manager: IManager, indexManager: IndexManager) { + public constructor( + manager: IManager, + indexManager: IndexManager + ) { this.manager = manager; this.response = manager.response; this.indexManager = indexManager; diff --git a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts index 6c64ca3c10e..a310ad80db0 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts @@ -6,27 +6,28 @@ export const createEnableIndexingTask = (params?: IElasticsearchTaskConfig) => { return createTaskDefinition({ id: "elasticsearchEnableIndexing", title: "Enable Indexing on Elasticsearch Indexes", - run: async ({ response, context, isAborted, isCloseToTimeout, input, store }) => { + run: async ({ response, context, isAborted, isCloseToTimeout, input, store, timer }) => { const { Manager } = await import( - /* webpackChunkName: "ElasticsearchTaskManager" */ + /* webpackChunkName: "Manager" */ "../Manager" ); const { IndexManager } = await import( - /* webpackChunkName: "ElasticsearchTaskSettings" */ "~/settings" + /* webpackChunkName: "IndexManager" */ "~/settings" ); const { EnableIndexingTaskRunner } = await import( - /* webpackChunkName: "ElasticsearchEnableIndexingTaskRunner" */ "./EnableIndexingTaskRunner" + /* webpackChunkName: "EnableIndexingTaskRunner" */ "./EnableIndexingTaskRunner" ); - const manager = new Manager({ + const manager = new Manager({ elasticsearchClient: params?.elasticsearchClient, documentClient: params?.documentClient, response, context, isAborted, isCloseToTimeout, - store + store, + timer }); const indexManager = new IndexManager( diff --git a/packages/api-elasticsearch-tasks/src/tasks/index.ts b/packages/api-elasticsearch-tasks/src/tasks/index.ts index 52680e600ca..61972c2d768 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/index.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/index.ts @@ -1,3 +1,4 @@ export * from "./enableIndexing"; +export * from "./dataSynchronization"; export * from "./reindexing"; export * from "./createIndexes"; diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts index 85f974b667f..908d8d911b7 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts @@ -1,5 +1,6 @@ import { IDynamoDbElasticsearchRecord, + IElasticsearchIndexingTaskValues, IElasticsearchIndexingTaskValuesKeys, IManager } from "~/types"; @@ -20,13 +21,16 @@ const getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | }; export class ReindexingTaskRunner { - private readonly manager: IManager; + private readonly manager: IManager; private keys?: IElasticsearchIndexingTaskValuesKeys; private readonly indexManager: IIndexManager; private readonly response: ITaskResponse; - public constructor(manager: IManager, indexManager: IndexManager) { + public constructor( + manager: IManager, + indexManager: IndexManager + ) { this.manager = manager; this.response = manager.response; this.indexManager = indexManager; diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts index 6194a69018d..851276356e2 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts @@ -5,26 +5,27 @@ export const createElasticsearchReindexingTask = (params?: IElasticsearchTaskCon return createTaskDefinition({ id: "elasticsearchReindexing", title: "Elasticsearch reindexing", - run: async ({ context, isCloseToTimeout, response, input, isAborted, store }) => { + run: async ({ context, isCloseToTimeout, response, input, isAborted, store, timer }) => { const { Manager } = await import( - /* webpackChunkName: "ElasticsearchReindexingManager" */ + /* webpackChunkName: "Manager" */ "../Manager" ); const { IndexManager } = await import( - /* webpackChunkName: "ElasticsearchReindexingSettings" */ "~/settings" + /* webpackChunkName: "IndexManager" */ "~/settings" ); const { ReindexingTaskRunner } = await import( - /* webpackChunkName: "ElasticsearchReindexingTaskRunner" */ "./ReindexingTaskRunner" + /* webpackChunkName: "ReindexingTaskRunner" */ "./ReindexingTaskRunner" ); - const manager = new Manager({ + const manager = new Manager({ elasticsearchClient: params?.elasticsearchClient, documentClient: params?.documentClient, response, context, isAborted, isCloseToTimeout, - store + store, + timer }); const indexManager = new IndexManager(manager.elasticsearch, input.settings || {}); diff --git a/packages/api-elasticsearch-tasks/src/types.ts b/packages/api-elasticsearch-tasks/src/types.ts index 2b832cc6621..abce57122f9 100644 --- a/packages/api-elasticsearch-tasks/src/types.ts +++ b/packages/api-elasticsearch-tasks/src/types.ts @@ -1,14 +1,20 @@ import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { Context as TasksContext } from "@webiny/tasks/types"; +import { + Context as TasksContext, + IIsCloseToTimeoutCallable, + ITaskResponseDoneResultOutput +} from "@webiny/tasks/types"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; import { Client } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; import { ITaskResponse } from "@webiny/tasks/response/abstractions"; import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb"; +import { ITimer } from "@webiny/handler-aws"; +import { Context as LoggerContext } from "@webiny/api-log/types"; -export interface Context extends ElasticsearchContext, TasksContext {} +export interface Context extends ElasticsearchContext, TasksContext, LoggerContext {} export interface IElasticsearchTaskConfig { documentClient?: DynamoDBDocument; @@ -51,15 +57,19 @@ export interface IDynamoDbElasticsearchRecord { modified: string; } -export interface IManager { +export interface IManager< + T, + O extends ITaskResponseDoneResultOutput = ITaskResponseDoneResultOutput +> { readonly documentClient: DynamoDBDocument; readonly elasticsearch: Client; readonly context: Context; readonly table: ReturnType; - readonly isCloseToTimeout: () => boolean; + readonly isCloseToTimeout: IIsCloseToTimeoutCallable; readonly isAborted: () => boolean; - readonly response: ITaskResponse; - readonly store: ITaskManagerStore; + readonly response: ITaskResponse; + readonly store: ITaskManagerStore; + readonly timer: ITimer; getEntity: (name: string) => Entity; diff --git a/packages/api-elasticsearch-tasks/tsconfig.build.json b/packages/api-elasticsearch-tasks/tsconfig.build.json index 014a0b3fcdf..b4a5c6bbf3a 100644 --- a/packages/api-elasticsearch-tasks/tsconfig.build.json +++ b/packages/api-elasticsearch-tasks/tsconfig.build.json @@ -3,8 +3,11 @@ "include": ["src"], "references": [ { "path": "../api/tsconfig.build.json" }, + { "path": "../api-dynamodb-to-elasticsearch/tsconfig.build.json" }, { "path": "../api-elasticsearch/tsconfig.build.json" }, + { "path": "../api-log/tsconfig.build.json" }, { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../db/tsconfig.build.json" }, { "path": "../db-dynamodb/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, { "path": "../tasks/tsconfig.build.json" }, diff --git a/packages/api-elasticsearch-tasks/tsconfig.json b/packages/api-elasticsearch-tasks/tsconfig.json index f2032353ca0..dfeac5eee47 100644 --- a/packages/api-elasticsearch-tasks/tsconfig.json +++ b/packages/api-elasticsearch-tasks/tsconfig.json @@ -3,8 +3,11 @@ "include": ["src", "__tests__"], "references": [ { "path": "../api" }, + { "path": "../api-dynamodb-to-elasticsearch" }, { "path": "../api-elasticsearch" }, + { "path": "../api-log" }, { "path": "../aws-sdk" }, + { "path": "../db" }, { "path": "../db-dynamodb" }, { "path": "../error" }, { "path": "../tasks" }, @@ -29,10 +32,16 @@ "~tests/*": ["./__tests__/*"], "@webiny/api/*": ["../api/src/*"], "@webiny/api": ["../api/src"], + "@webiny/api-dynamodb-to-elasticsearch/*": ["../api-dynamodb-to-elasticsearch/src/*"], + "@webiny/api-dynamodb-to-elasticsearch": ["../api-dynamodb-to-elasticsearch/src"], "@webiny/api-elasticsearch/*": ["../api-elasticsearch/src/*"], "@webiny/api-elasticsearch": ["../api-elasticsearch/src"], + "@webiny/api-log/*": ["../api-log/src/*"], + "@webiny/api-log": ["../api-log/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], "@webiny/aws-sdk": ["../aws-sdk/src"], + "@webiny/db/*": ["../db/src/*"], + "@webiny/db": ["../db/src"], "@webiny/db-dynamodb/*": ["../db-dynamodb/src/*"], "@webiny/db-dynamodb": ["../db-dynamodb/src"], "@webiny/error/*": ["../error/src/*"], diff --git a/packages/api-elasticsearch/src/cursors.ts b/packages/api-elasticsearch/src/cursors.ts index d6c116d1fd4..a6984671259 100644 --- a/packages/api-elasticsearch/src/cursors.ts +++ b/packages/api-elasticsearch/src/cursors.ts @@ -3,12 +3,16 @@ import { PrimitiveValue } from "~/types"; /** * Encode a received cursor value into something that can be passed on to the user. */ -export const encodeCursor = (cursor?: string | string[] | null): string | undefined => { - if (!cursor) { +export const encodeCursor = (input?: PrimitiveValue[]): string | undefined => { + if (!input) { return undefined; } - cursor = Array.isArray(cursor) ? cursor.map(encodeURIComponent) : encodeURIComponent(cursor); + const cursor = Array.isArray(input) + ? input + .filter((item: PrimitiveValue): item is string | number | boolean => item !== null) + .map(item => encodeURIComponent(item)) + : encodeURIComponent(input); try { return Buffer.from(JSON.stringify(cursor)).toString("base64"); @@ -28,7 +32,7 @@ export const decodeCursor = (cursor?: string | null): PrimitiveValue[] | undefin try { const value = JSON.parse(Buffer.from(cursor, "base64").toString("ascii")); if (Array.isArray(value)) { - return value.map(decodeURIComponent); + return value.filter(item => item !== null).map(decodeURIComponent); } const decoded = decodeURIComponent(value); return decoded ? [decoded] : undefined; diff --git a/packages/api-elasticsearch/src/types.ts b/packages/api-elasticsearch/src/types.ts index 8b0ac7c5dde..a57bae522ea 100644 --- a/packages/api-elasticsearch/src/types.ts +++ b/packages/api-elasticsearch/src/types.ts @@ -1,6 +1,6 @@ -import { Client, ApiResponse } from "@elastic/elasticsearch"; -import { BoolQueryConfig as esBoolQueryConfig, Query as esQuery } from "elastic-ts"; -import { Context } from "@webiny/api/types"; +import { ApiResponse, Client } from "@elastic/elasticsearch"; +import { BoolQueryConfig, PrimitiveValue, Query as esQuery } from "elastic-ts"; +import { Context, GenericRecord } from "@webiny/api/types"; /** * Re-export some dep lib types. */ @@ -15,7 +15,7 @@ export interface ElasticsearchContext extends Context { * To simplify our plugins, we say that query contains arrays of objects, not single objects. * And that they all are defined as empty arrays at the start. */ -export interface ElasticsearchBoolQueryConfig extends esBoolQueryConfig { +export interface ElasticsearchBoolQueryConfig extends BoolQueryConfig { must: esQuery[]; filter: esQuery[]; should: esQuery[]; @@ -77,29 +77,40 @@ export interface ElasticsearchQueryBuilderArgsPlugin { * Elasticsearch responses. */ export interface ElasticsearchSearchResponseHit { + _index: string; + _type: string; + _id: string; + _score: number | null; _source: T; - sort: string; + sort: PrimitiveValue[]; } export interface ElasticsearchSearchResponseAggregationBucket { key: T; doc_count: number; } -export interface ElasticsearchSearchResponse { - body: { - hits: { - hits: ElasticsearchSearchResponseHit[]; - total: { - value: number; - }; - }; - aggregations: { - [key: string]: { - buckets: ElasticsearchSearchResponseAggregationBucket[]; - }; - }; + +export interface ElasticsearchSearchResponseBodyHits { + hits: ElasticsearchSearchResponseHit[]; + total: { + value: number; + }; +} + +export interface ElasticsearchSearchResponseBodyAggregations { + [key: string]: { + buckets: ElasticsearchSearchResponseAggregationBucket[]; }; } +export interface ElasticsearchSearchResponseBody { + hits: ElasticsearchSearchResponseBodyHits; + aggregations: ElasticsearchSearchResponseBodyAggregations; +} + +export interface ElasticsearchSearchResponse { + body: ElasticsearchSearchResponseBody; +} + export interface ElasticsearchIndexRequestBodyMappingsDynamicTemplate { [key: string]: { path_match?: string; diff --git a/packages/api-form-builder-so-ddb-es/src/index.ts b/packages/api-form-builder-so-ddb-es/src/index.ts index 1968a95b18d..33773290324 100644 --- a/packages/api-form-builder-so-ddb-es/src/index.ts +++ b/packages/api-form-builder-so-ddb-es/src/index.ts @@ -139,6 +139,27 @@ export const createFormBuilderStorageOperations: FormBuilderStorageOperationsFac return { beforeInit: async (context: FormBuilderContext) => { + context.db.registry.register({ + item: entities.form, + app: "fb", + tags: ["regular", "form", entities.form.name] + }); + context.db.registry.register({ + item: entities.esForm, + app: "fb", + tags: ["es", "form", entities.esForm.name] + }); + context.db.registry.register({ + item: entities.submission, + app: "fb", + tags: ["regular", "form-submission", entities.submission.name] + }); + context.db.registry.register({ + item: entities.esSubmission, + app: "fb", + tags: ["es", "form-submission", entities.esSubmission.name] + }); + const types: string[] = [ // Elasticsearch CompressionPlugin.type, diff --git a/packages/api-form-builder-so-ddb-es/src/types.ts b/packages/api-form-builder-so-ddb-es/src/types.ts index 8fbba21e172..8b9a96a8289 100644 --- a/packages/api-form-builder-so-ddb-es/src/types.ts +++ b/packages/api-form-builder-so-ddb-es/src/types.ts @@ -1,26 +1,24 @@ import { + FormBuilder, + FormBuilderContext as BaseFormBuilderContext, + FormBuilderFormStorageOperations as BaseFormBuilderFormStorageOperations, + FormBuilderSettingsStorageOperations as BaseFormBuilderSettingsStorageOperations, FormBuilderStorageOperations as BaseFormBuilderStorageOperations, - FormBuilderSystemStorageOperations as BaseFormBuilderSystemStorageOperations, FormBuilderSubmissionStorageOperations as BaseFormBuilderSubmissionStorageOperations, - FormBuilderSettingsStorageOperations as BaseFormBuilderSettingsStorageOperations, - FormBuilderFormStorageOperations as BaseFormBuilderFormStorageOperations, - FormBuilderContext + FormBuilderSystemStorageOperations as BaseFormBuilderSystemStorageOperations } from "@webiny/api-form-builder/types"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; -import { AttributeDefinition } from "@webiny/db-dynamodb/toolbox"; +import { AttributeDefinition, Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; import { PluginCollection } from "@webiny/plugins/types"; -export { FormBuilderContext }; - export type Attributes = Record; export enum ENTITIES { FORM = "FormBuilderForm", + ES_SUBMISSION = "FormBuilderSubmissionEs", ES_FORM = "FormBuilderFormEs", SUBMISSION = "FormBuilderSubmission", - ES_SUBMISSION = "FormBuilderSubmissionEs", SYSTEM = "FormBuilderSystem", SETTINGS = "FormBuilderSettings" } @@ -96,3 +94,9 @@ export interface FormBuilderStorageOperations export interface FormBuilderStorageOperationsFactory { (params: FormBuilderStorageOperationsFactoryParams): FormBuilderStorageOperations; } + +export interface FormBuilderContext extends BaseFormBuilderContext { + formBuilder: FormBuilder & { + storageOperations: FormBuilderStorageOperations; + }; +} diff --git a/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts b/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts index 8165f7be399..b2055083d59 100644 --- a/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts +++ b/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts @@ -23,10 +23,9 @@ export const createEventBridgeHandler = () => { /** * Since the event is at the infrastructure level, it has no knowledge about tenancy. - * We loop through all tenants in the system and trigger the "EmptyTrashBins" task. + * We trigger the `hcmsEntriesEmptyTrashBins` using root tenant. */ - const tenants = await context.tenancy.listTenants(); - await context.tenancy.withEachTenant(tenants, async () => { + await context.tenancy.withRootTenant(async () => { await context.tasks.trigger({ definition: "hcmsEntriesEmptyTrashBins" }); diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts index d2270120bfb..1f10e4b1dfe 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts @@ -11,7 +11,10 @@ export interface CreateBulkActionGraphQL { export const createBulkActionGraphQL = (config: CreateBulkActionGraphQL) => { return new ContextPlugin(async context => { - if (!(await isHeadlessCmsReady(context))) { + const tenant = context.tenancy.getCurrentTenant(); + const locale = context.i18n.getContentLocale(); + + if (!locale || !(await isHeadlessCmsReady(context))) { return; } @@ -62,7 +65,10 @@ export const createBulkActionGraphQL = (config: CreateBulkActionGraphQL) => { }); } } - } + }, + isApplicable: context => + context.tenancy.getCurrentTenant().id === tenant.id && + context.i18n.getContentLocale()?.code === locale.code }); plugin.name = `headless-cms.graphql.schema.bulkAction.${model.modelId}.${config.name}`; diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts index 40aa2d64e50..e6d8690ecf6 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts @@ -5,7 +5,10 @@ import { CMS_MODEL_SINGLETON_TAG } from "@webiny/api-headless-cms/constants"; export const createDefaultGraphQL = () => { return new ContextPlugin(async context => { - if (!(await isHeadlessCmsReady(context))) { + const tenant = context.tenancy.getCurrentTenant(); + const locale = context.i18n.getContentLocale(); + + if (!locale || !(await isHeadlessCmsReady(context))) { return; } @@ -54,7 +57,10 @@ export const createDefaultGraphQL = () => { data: JSON ): BulkActionResponse } - ` + `, + isApplicable: context => + context.tenancy.getCurrentTenant().id === tenant.id && + context.i18n.getContentLocale()?.code === locale.code }); plugin.name = `headless-cms.graphql.schema.bulkAction.default.${model.modelId}`; diff --git a/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts b/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts index 3cdab9b7ca4..71bd89fb49a 100644 --- a/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts +++ b/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts @@ -1,10 +1,11 @@ -import { createPrivateTaskDefinition, TaskDataStatus } from "@webiny/tasks"; +import { createTaskDefinition } from "@webiny/tasks"; +import { createDeleteEntry, createListDeletedEntries } from "~/useCases"; import { HcmsBulkActionsContext, - IBulkActionOperationByModelInput, - TrashBinCleanUpParams + IEmptyTrashBinsInput, + IEmptyTrashBinsOutput, + IEmptyTrashBinsTaskParams } from "~/types"; -import { ChildTasksCleanup } from "~/useCases/internals"; const calculateDateTimeString = () => { // Retrieve the retention period from the environment variable WEBINY_TRASH_BIN_RETENTION_PERIOD_DAYS, @@ -23,122 +24,95 @@ const calculateDateTimeString = () => { return currentDate.toISOString(); }; -const cleanup = async ({ context, task }: TrashBinCleanUpParams) => { - // We want to clean all child tasks and logs, which have no errors. - const childTasksCleanup = new ChildTasksCleanup(); - try { - await childTasksCleanup.execute({ - context, - task - }); - } catch (ex) { - console.error(`Error while cleaning "EmptyTrashBins" child tasks.`, ex); - } -}; - export const createEmptyTrashBinsTask = () => { - return createPrivateTaskDefinition({ + return createTaskDefinition< + HcmsBulkActionsContext, + IEmptyTrashBinsInput, + IEmptyTrashBinsOutput + >({ + isPrivate: true, id: "hcmsEntriesEmptyTrashBins", title: "Headless CMS - Empty all trash bins", - description: - "Delete all entries found in the trash bin, for each model found in the system.", - maxIterations: 24, + description: "Delete all entries in the trash bin for each model in the system.", + maxIterations: 120, disableDatabaseLogs: true, - run: async params => { - const { response, isAborted, isCloseToTimeout, context, trigger, input, store } = - params; + run: async (params: IEmptyTrashBinsTaskParams) => { + const { response, isAborted, context, input, isCloseToTimeout } = params; + + // Abort the task if needed. if (isAborted()) { return response.aborted(); - } else if (isCloseToTimeout()) { - return response.continue( - { - ...input - }, - { - seconds: 30 - } - ); } - if (input.triggered) { - const { items } = await context.tasks.listTasks({ - where: { - parentId: store.getTask().id, - taskStatus_in: [TaskDataStatus.RUNNING, TaskDataStatus.PENDING] - }, - limit: 100000 - }); + // Fetch all tenants, excluding those already processed. + const baseTenants = await context.tenancy.listTenants(); + const executedTenantIds = input.executedTenantIds || []; + const tenants = baseTenants.filter(tenant => !executedTenantIds.includes(tenant.id)); + let shouldContinue = false; // Flag to check if task should continue. - if (items.length === 0) { - return response.done( - "Task done: emptying the trash bin for all registered models." - ); + // Iterate over each tenant. + await context.tenancy.withEachTenant(tenants, async tenant => { + if (isCloseToTimeout()) { + shouldContinue = true; + return; } - for (const item of items) { - const status = await context.tasks.fetchServiceInfo(item.id); - - if (status?.status === "FAILED" || status?.status === "TIMED_OUT") { - await context.tasks.updateTask(item.id, { - taskStatus: TaskDataStatus.FAILED - }); - continue; - } - - if (status?.status === "ABORTED") { - await context.tasks.updateTask(item.id, { - taskStatus: TaskDataStatus.ABORTED - }); + // Fetch all locales for the tenant. + const locales = context.i18n.getLocales(); + await context.i18n.withEachLocale(locales, async () => { + if (isCloseToTimeout()) { + shouldContinue = true; + return; } - } - return response.continue( - { - ...input - }, - { - seconds: 3600 - } - ); - } + // List all non-private models for the current locale. + const models = await context.security.withoutAuthorization(async () => + (await context.cms.listModels()).filter(m => !m.isPrivate) + ); - try { - const locales = context.i18n.getLocales(); + // Process each model to delete trashed entries. + for (const model of models) { + const list = createListDeletedEntries(context); // List trashed entries. + const mutation = createDeleteEntry(context); // Mutation to delete entries. - await context.i18n.withEachLocale(locales, async () => { - const models = await context.security.withoutAuthorization(async () => { - return (await context.cms.listModels()).filter(model => !model.isPrivate); - }); + // Query parameters for fetching deleted entries older than a minute ago. + const listEntriesParams = { + where: { deletedOn_lt: calculateDateTimeString() }, + limit: 50 + }; - for (const model of models) { - await trigger({ - name: `Headless CMS - Empty trash bin for "${model.name}" model.`, - definition: "hcmsBulkListDeleteEntries", - input: { - modelId: model.modelId, - where: { - deletedOn_lt: calculateDateTimeString() + let result; + // Continue deleting entries while there are entries left to delete. + while ( + (result = await list.execute(model.modelId, listEntriesParams)) && + result.meta.totalCount > 0 + ) { + if (isCloseToTimeout()) { + shouldContinue = true; + break; + } + for (const entry of result.entries) { + if (isCloseToTimeout()) { + shouldContinue = true; + break; } + // Delete each entry individually. + await mutation.execute(model, entry.id); } - }); + } } }); - return response.continue( - { - triggered: true - }, - { - seconds: 120 - } - ); - } catch (ex) { - return response.error(ex.message ?? "Error while executing EmptyTrashBins task"); - } - }, - onMaxIterations: cleanup, - onDone: cleanup, - onError: cleanup, - onAbort: cleanup + // If the task isn't continuing, add the tenant to the executed list. + if (!shouldContinue) { + executedTenantIds.push(tenant.id); + } + }); + + // Continue the task or mark it as done based on the `shouldContinue` flag. + return shouldContinue + ? response.continue({ ...input, executedTenantIds }) + : response.done("Task done: emptied the trash bin for all registered models."); + } }); }; diff --git a/packages/api-headless-cms-bulk-actions/src/types.ts b/packages/api-headless-cms-bulk-actions/src/types.ts index 913818c8b75..2ba5e556580 100644 --- a/packages/api-headless-cms-bulk-actions/src/types.ts +++ b/packages/api-headless-cms-bulk-actions/src/types.ts @@ -2,10 +2,6 @@ import { CmsContext } from "@webiny/api-headless-cms/types"; import { Context as BaseContext } from "@webiny/handler/types"; import { Context as TasksContext, - ITaskOnAbortParams, - ITaskOnErrorParams, - ITaskOnMaxIterationsParams, - ITaskOnSuccessParams, ITaskResponseDoneResultOutput, ITaskRunParams } from "@webiny/tasks/types"; @@ -70,11 +66,17 @@ export type IBulkActionOperationByModelTaskParams = ITaskRunParams< >; /** - * Trash Bin + * Empty Trash Bin */ -export type TrashBinCleanUpParams = - | ITaskOnSuccessParams - | ITaskOnErrorParams - | ITaskOnAbortParams - | ITaskOnMaxIterationsParams; +export interface IEmptyTrashBinsInput { + executedTenantIds?: string[] | null; +} + +export type IEmptyTrashBinsOutput = ITaskResponseDoneResultOutput; + +export type IEmptyTrashBinsTaskParams = ITaskRunParams< + HcmsBulkActionsContext, + IEmptyTrashBinsInput, + IEmptyTrashBinsOutput +>; diff --git a/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js b/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js index e657c72bbb3..4988c78bb75 100644 --- a/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js +++ b/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js @@ -66,11 +66,19 @@ module.exports = () => { }); }); + const initializedDbPlugins = dbPlugins({ + table: process.env.DB_TABLE, + driver: new DynamoDbDriver({ + documentClient + }) + }); + return { storageOperations: createStorageOperations({ documentClient, elasticsearch: elasticsearchClient, plugins: [ + ...initializedDbPlugins, getElasticsearchOperators(), createCmsEntryElasticsearchBodyModifierPlugin({ modifyBody: ({ body }) => { @@ -91,16 +99,7 @@ module.exports = () => { }) ] }), - plugins: [ - ...plugins, - dbPlugins({ - table: process.env.DB_TABLE, - driver: new DynamoDbDriver({ - documentClient - }) - }), - createOrRefreshIndexSubscription - ] + plugins: [...plugins, ...initializedDbPlugins, createOrRefreshIndexSubscription] }; }); }; diff --git a/packages/api-headless-cms-ddb-es/src/index.ts b/packages/api-headless-cms-ddb-es/src/index.ts index b4bf759c8c1..7c0dbad00e8 100644 --- a/packages/api-headless-cms-ddb-es/src/index.ts +++ b/packages/api-headless-cms-ddb-es/src/index.ts @@ -124,6 +124,16 @@ export const createStorageOperations: StorageOperationsFactory = params => { return { name: "dynamodb:elasticsearch", beforeInit: async context => { + context.db.registry.register({ + item: entities.entries, + app: "cms", + tags: ["regular", entities.entries.name] + }); + context.db.registry.register({ + item: entities.entriesEs, + app: "cms", + tags: ["es", entities.entriesEs.name] + }); /** * Attach the elasticsearch into context if it is not already attached. */ diff --git a/packages/api-headless-cms-ddb-es/src/types.ts b/packages/api-headless-cms-ddb-es/src/types.ts index 7618bf64df4..e7fae5a605f 100644 --- a/packages/api-headless-cms-ddb-es/src/types.ts +++ b/packages/api-headless-cms-ddb-es/src/types.ts @@ -7,14 +7,14 @@ import { CmsModelField, CmsModelFieldToGraphQLPlugin, CmsModelFieldType, + HeadlessCms, HeadlessCmsStorageOperations as BaseHeadlessCmsStorageOperations } from "@webiny/api-headless-cms/types"; -import { TableConstructor } from "@webiny/db-dynamodb/toolbox"; +import { AttributeDefinition, Entity, Table, TableConstructor } from "@webiny/db-dynamodb/toolbox"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { AttributeDefinition } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { PluginsContainer } from "@webiny/plugins"; +import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; /** * A definition of the entry that is being prepared for the Elasticsearch. @@ -167,6 +167,12 @@ export interface StorageOperationsFactoryParams { plugins?: PluginCollection; } +export interface CmsContext extends BaseCmsContext, ElasticsearchContext { + cms: HeadlessCms & { + storageOperations: HeadlessCmsStorageOperations; + }; +} + export interface HeadlessCmsStorageOperations extends BaseHeadlessCmsStorageOperations { getTable: () => Table; getEsTable: () => Table; @@ -180,10 +186,6 @@ export interface StorageOperationsFactory { (params: StorageOperationsFactoryParams): HeadlessCmsStorageOperations; } -export interface CmsContext extends BaseCmsContext { - [key: string]: any; -} - export interface CmsEntryStorageOperations extends BaseCmsEntryStorageOperations { dataLoaders: DataLoadersHandlerInterface; } diff --git a/packages/api-headless-cms-import-export/src/graphql/index.ts b/packages/api-headless-cms-import-export/src/graphql/index.ts index a18a1e07407..b021c9ff419 100644 --- a/packages/api-headless-cms-import-export/src/graphql/index.ts +++ b/packages/api-headless-cms-import-export/src/graphql/index.ts @@ -7,15 +7,20 @@ import type { NonEmptyArray } from "@webiny/api/types"; import { CmsModel } from "@webiny/api-headless-cms/types"; export const attachHeadlessCmsImportExportGraphQL = async (context: Context): Promise => { + const tenant = context.tenancy.getCurrentTenant(); + const locale = context.i18n.getContentLocale(); const models = await listModels(context); - if (models.length === 0) { + if (!locale || models.length === 0) { return; } const plugin = new CmsGraphQLSchemaPlugin({ typeDefs: createTypeDefs(models as NonEmptyArray), - resolvers: createResolvers(models as NonEmptyArray) + resolvers: createResolvers(models as NonEmptyArray), + isApplicable: context => + context.tenancy.getCurrentTenant().id === tenant.id && + context.i18n.getContentLocale()?.code === locale.code }); plugin.name = "headlessCms.graphql.importExport"; diff --git a/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts b/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts index e457e149ee1..b1a6c69aeac 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts @@ -1,10 +1,11 @@ import { mdbid } from "@webiny/utils"; import models from "./mocks/contentModels"; import { useGraphQLHandler } from "../testHelpers/useGraphQLHandler"; -import { CmsContext, CmsEntry, CmsGroup, CmsModel, StorageOperationsCmsModel } from "~/types"; +import { CmsEntry, CmsGroup, CmsModel, StorageOperationsCmsModel } from "~/types"; import { useCategoryManageHandler } from "../testHelpers/useCategoryManageHandler"; import { useCategoryReadHandler } from "../testHelpers/useCategoryReadHandler"; import { useProductManageHandler } from "../testHelpers/useProductManageHandler"; +import { createStorageOperationsContext } from "~tests/storageOperations/context"; const cliPackageJson = require("@webiny/cli/package.json"); const webinyVersion = cliPackageJson.version; @@ -288,9 +289,11 @@ describe("Republish entries", () => { const { storageOperations, plugins } = useCategoryManageHandler(manageOpts); - await storageOperations.beforeInit({ - plugins - } as CmsContext); + await storageOperations.beforeInit( + await createStorageOperationsContext({ + plugins + }) + ); const { entry: galaEntry } = createEntry(productModel, { title: "Gala", diff --git a/packages/api-headless-cms/__tests__/storageOperations/context.ts b/packages/api-headless-cms/__tests__/storageOperations/context.ts new file mode 100644 index 00000000000..f0fe3755da9 --- /dev/null +++ b/packages/api-headless-cms/__tests__/storageOperations/context.ts @@ -0,0 +1,34 @@ +import dbPlugins from "@webiny/handler-db"; +import { PluginsContainer } from "@webiny/plugins"; +import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { DynamoDbDriver } from "@webiny/db-dynamodb"; +import { CmsContext } from "~/types"; +import { Context } from "@webiny/api"; + +export interface ICreateStorageOperationsContextParams { + plugins?: PluginsContainer; +} + +export const createStorageOperationsContext = async ( + params: ICreateStorageOperationsContextParams +): Promise => { + const dbPluginsInitialized = dbPlugins({ + table: process.env.DB_TABLE, + driver: new DynamoDbDriver({ + documentClient: getDocumentClient() + }) + }); + const plugins = params.plugins || new PluginsContainer([]); + plugins.register(...dbPluginsInitialized); + + const context = new Context({ + plugins, + WEBINY_VERSION: "0.0.0" + }) as unknown as CmsContext; + + for (const db of dbPluginsInitialized) { + await db.apply(context); + } + + return context; +}; diff --git a/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts b/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts index 6920c5331c5..a0879ab63f3 100644 --- a/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts +++ b/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts @@ -1,6 +1,6 @@ import { createPersonEntries, createPersonModel, deletePersonModel } from "./helpers"; import { useGraphQLHandler } from "../testHelpers/useGraphQLHandler"; -import { CmsContext } from "~/types"; +import { createStorageOperationsContext } from "~tests/storageOperations/context"; jest.setTimeout(90000); @@ -15,9 +15,11 @@ describe("Entries storage operations", () => { * Some others might not need them... */ beforeAll(async () => { - await storageOperations.beforeInit({ - plugins - } as unknown as CmsContext); + await storageOperations.beforeInit( + await createStorageOperationsContext({ + plugins + }) + ); }); beforeEach(async () => { diff --git a/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts b/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts index 6e49c1b6206..4d358ef2a7f 100644 --- a/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts +++ b/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts @@ -1,10 +1,10 @@ -import { CmsContext } from "~/types"; import { createPersonEntries, createPersonModel, deletePersonModel } from "~tests/storageOperations/helpers"; import { useGraphQLHandler } from "~tests/testHelpers/useGraphQLHandler"; +import { createStorageOperationsContext } from "~tests/storageOperations/context"; describe("field unique values listing", () => { const { storageOperations, plugins } = useGraphQLHandler({ @@ -17,9 +17,11 @@ describe("field unique values listing", () => { * Some others might not need them... */ beforeAll(async () => { - await storageOperations.beforeInit({ - plugins - } as unknown as CmsContext); + await storageOperations.beforeInit( + await createStorageOperationsContext({ + plugins + }) + ); }); beforeEach(async () => { diff --git a/packages/api-headless-cms/package.json b/packages/api-headless-cms/package.json index 363b6d99a80..45ba373392c 100644 --- a/packages/api-headless-cms/package.json +++ b/packages/api-headless-cms/package.json @@ -52,6 +52,8 @@ "@webiny/api-wcp": "0.0.0", "@webiny/aws-sdk": "0.0.0", "@webiny/cli": "0.0.0", + "@webiny/db-dynamodb": "0.0.0", + "@webiny/handler-db": "0.0.0", "@webiny/project-utils": "0.0.0", "apollo-graphql": "^0.9.5", "get-yarn-workspaces": "^1.0.2", diff --git a/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts b/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts index 1ec1e69f229..a10bb3f6373 100644 --- a/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts +++ b/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts @@ -232,6 +232,7 @@ const createGraphQLSchema = async (params: CreateGraphQLSchemaParams): Promise(CmsGraphQLSchemaPlugin.type) + .filter(plugin => plugin.isApplicable(context)) .reduce>((collection, plugin) => { const name = plugin.name || `${CmsGraphQLSchemaPlugin.type}-${generateAlphaNumericId(16)}`; diff --git a/packages/api-headless-cms/tsconfig.build.json b/packages/api-headless-cms/tsconfig.build.json index 777323f7d4c..3062670240e 100644 --- a/packages/api-headless-cms/tsconfig.build.json +++ b/packages/api-headless-cms/tsconfig.build.json @@ -17,7 +17,8 @@ { "path": "../utils/tsconfig.build.json" }, { "path": "../validation/tsconfig.build.json" }, { "path": "../api-wcp/tsconfig.build.json" }, - { "path": "../aws-sdk/tsconfig.build.json" } + { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../db-dynamodb/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", diff --git a/packages/api-headless-cms/tsconfig.json b/packages/api-headless-cms/tsconfig.json index ec25eab3be4..06ba7176b3a 100644 --- a/packages/api-headless-cms/tsconfig.json +++ b/packages/api-headless-cms/tsconfig.json @@ -17,7 +17,8 @@ { "path": "../utils" }, { "path": "../validation" }, { "path": "../api-wcp" }, - { "path": "../aws-sdk" } + { "path": "../aws-sdk" }, + { "path": "../db-dynamodb" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], @@ -57,7 +58,9 @@ "@webiny/api-wcp/*": ["../api-wcp/src/*"], "@webiny/api-wcp": ["../api-wcp/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], - "@webiny/aws-sdk": ["../aws-sdk/src"] + "@webiny/aws-sdk": ["../aws-sdk/src"], + "@webiny/db-dynamodb/*": ["../db-dynamodb/src/*"], + "@webiny/db-dynamodb": ["../db-dynamodb/src"] }, "baseUrl": "." } diff --git a/packages/api-page-builder-so-ddb-es/__tests__/__api__/setupFile.js b/packages/api-page-builder-so-ddb-es/__tests__/__api__/setupFile.js index bee672d749b..d6700c29a64 100644 --- a/packages/api-page-builder-so-ddb-es/__tests__/__api__/setupFile.js +++ b/packages/api-page-builder-so-ddb-es/__tests__/__api__/setupFile.js @@ -35,22 +35,22 @@ module.exports = () => { } }); + const initializedDbPlugins = dbPlugins({ + table: process.env.DB_TABLE, + driver: new DynamoDbDriver({ + documentClient + }) + }); + return { storageOperations: createStorageOperations({ documentClient, elasticsearch: elasticsearchClient, table: table => ({ ...table, name: process.env.DB_TABLE }), - esTable: table => ({ ...table, name: process.env.DB_TABLE_ELASTICSEARCH }) + esTable: table => ({ ...table, name: process.env.DB_TABLE_ELASTICSEARCH }), + plugins: [...initializedDbPlugins] }), - plugins: [ - ...plugins, - dbPlugins({ - table: process.env.DB_TABLE, - driver: new DynamoDbDriver({ - documentClient - }) - }) - ] + plugins: [...plugins, ...initializedDbPlugins] }; }); }; diff --git a/packages/api-page-builder-so-ddb-es/src/index.ts b/packages/api-page-builder-so-ddb-es/src/index.ts index ff9a54f69f2..73d8f53705d 100644 --- a/packages/api-page-builder-so-ddb-es/src/index.ts +++ b/packages/api-page-builder-so-ddb-es/src/index.ts @@ -208,6 +208,16 @@ export const createStorageOperations: StorageOperationsFactory = params => { return { beforeInit: async (context: PbContext) => { + context.db.registry.register({ + item: entities.pages, + app: "pb", + tags: ["regular", entities.pages.name] + }); + context.db.registry.register({ + item: entities.pagesEs, + app: "pb", + tags: ["es", entities.pagesEs.name] + }); const types: string[] = [ // Elasticsearch CompressionPlugin.type, diff --git a/packages/api-page-builder-so-ddb-es/src/types.ts b/packages/api-page-builder-so-ddb-es/src/types.ts index 6201f33b364..f0721d7cfbd 100644 --- a/packages/api-page-builder-so-ddb-es/src/types.ts +++ b/packages/api-page-builder-so-ddb-es/src/types.ts @@ -2,18 +2,15 @@ import { BlockCategoryStorageOperations as BaseBlockCategoryStorageOperations, CategoryStorageOperations as BaseCategoryStorageOperations, PageBlockStorageOperations as BasePageBlockStorageOperations, + PageBuilderContextObject, PageBuilderStorageOperations as BasePageBuilderStorageOperations, PageTemplateStorageOperations as BasePageTemplateStorageOperations, - PbContext + PbContext as BasePbContext } from "@webiny/api-page-builder/types"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import { AttributeDefinition, Entity, Table, TableConstructor } from "@webiny/db-dynamodb/toolbox"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; import { Client } from "@elastic/elasticsearch"; import { PluginCollection } from "@webiny/plugins/types"; -import { TableConstructor } from "@webiny/db-dynamodb/toolbox"; -import { AttributeDefinition } from "@webiny/db-dynamodb/toolbox"; - -export { PbContext }; export type Attributes = Record; @@ -52,6 +49,12 @@ export interface PageBuilderStorageOperations extends BasePageBuilderStorageOper >; } +export interface PbContext extends BasePbContext { + pageBuilder: PageBuilderContextObject & { + storageOperations: PageBuilderStorageOperations; + }; +} + export interface StorageOperationsFactoryParams { documentClient: DynamoDBDocument; elasticsearch: Client; diff --git a/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts b/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts index 4cc5ba32003..6859668ffa3 100644 --- a/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts +++ b/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts @@ -12,21 +12,11 @@ export interface GroupsTeamsAuthorizerConfig Promise | GroupSlug; - /** - * List group slugs to load permissions from. - */ - listGroupSlugs?: (context: TContext) => Promise | GroupSlug[]; - - /** - * List team slugs to load groups and ultimately permissions from. - */ - listTeamSlugs?: (context: TContext) => Promise | TeamSlug[]; - /** * If a security group is not found, try loading it from a parent tenant (default: true). */ @@ -65,11 +55,6 @@ export const listPermissionsFromGroupsAndTeams = async < groupSlugs.push(loadedGroupSlug); } - if (config.listGroupSlugs) { - const loadedGroupSlugs = await config.listGroupSlugs(context); - groupSlugs.push(...loadedGroupSlugs); - } - if (identity.group) { groupSlugs.push(identity.group); } @@ -80,11 +65,6 @@ export const listPermissionsFromGroupsAndTeams = async < if (wcp.canUseTeams()) { // Load groups coming from teams. - if (config.listTeamSlugs) { - const loadedTeamSlugs = await config.listTeamSlugs(context); - teamSlugs.push(...loadedTeamSlugs); - } - if (identity.team) { teamSlugs.push(identity.team); } diff --git a/packages/app-headless-cms-common/src/createFieldsList.ts b/packages/app-headless-cms-common/src/createFieldsList.ts index 87a91ca75dd..f7228d2385d 100644 --- a/packages/app-headless-cms-common/src/createFieldsList.ts +++ b/packages/app-headless-cms-common/src/createFieldsList.ts @@ -1,4 +1,4 @@ -import { CmsModelField, CmsModelFieldTypePlugin, CmsModel } from "~/types"; +import { CmsModel, CmsModelField, CmsModelFieldTypePlugin } from "~/types"; import { plugins } from "@webiny/plugins"; interface CreateFieldsListParams { @@ -9,7 +9,7 @@ interface CreateFieldsListParams { export function createFieldsList({ model, - fields, + fields: inputFields, graphQLTypePrefix }: CreateFieldsListParams): string { const fieldPlugins: Record = plugins @@ -18,7 +18,7 @@ export function createFieldsList({ const typePrefix = graphQLTypePrefix ?? model.singularApiName; - const allFields = fields + const fields = inputFields .map(field => { if (!fieldPlugins[field.type]) { console.log(`Unknown field plugin for field type "${field.type}".`); @@ -46,14 +46,11 @@ export function createFieldsList({ return field.fieldId; }) .filter(Boolean); - /** - * If there are no fields for a given type, we add a dummy `_empty` field, which will also be present in the schema - * on the API side, to protect the schema from invalid types. + * If there are no fields, let's always load the `id` field. */ - if (!allFields.length) { - allFields.push("_empty"); + if (fields.length === 0) { + fields.push("id"); } - - return allFields.join("\n"); + return fields.join("\n"); } diff --git a/packages/app-headless-cms-common/src/entries.graphql.ts b/packages/app-headless-cms-common/src/entries.graphql.ts index 8512cb63058..6e5be56d9b7 100644 --- a/packages/app-headless-cms-common/src/entries.graphql.ts +++ b/packages/app-headless-cms-common/src/entries.graphql.ts @@ -112,36 +112,6 @@ const createEntrySystemFields = (model: CmsModel) => { type displayName } - revisionCreatedOn - revisionSavedOn - revisionModifiedOn - revisionFirstPublishedOn - revisionLastPublishedOn - revisionCreatedBy { - id - type - displayName - } - revisionSavedBy { - id - type - displayName - } - revisionModifiedBy { - id - type - displayName - } - revisionFirstPublishedBy { - id - type - displayName - } - revisionLastPublishedBy { - id - type - displayName - } ${optionalFields} `; }; diff --git a/packages/app-page-builder-elements/src/modifiers/styles/border.ts b/packages/app-page-builder-elements/src/modifiers/styles/border.ts index 3f731eb3ead..a02ab2bdb26 100644 --- a/packages/app-page-builder-elements/src/modifiers/styles/border.ts +++ b/packages/app-page-builder-elements/src/modifiers/styles/border.ts @@ -45,10 +45,10 @@ const border: ElementStylesModifier = ({ element, theme }) => { if (radius) { if (radius.advanced) { Object.assign(styles, { - borderRadiusTop: radius.top && parseInt(radius.top), - borderRadiusRight: radius.right && parseInt(radius.right), - borderRadiusBottom: radius.bottom && parseInt(radius.bottom), - borderRadiusLeft: radius.left && parseInt(radius.left) + borderTopLeftRadius: radius.topLeft && parseInt(radius.topLeft), + borderTopRightRadius: radius.topRight && parseInt(radius.topRight), + borderBottomLeftRadius: radius.bottomLeft && parseInt(radius.bottomLeft), + borderBottomRightRadius: radius.bottomRight && parseInt(radius.bottomRight) }); } else { Object.assign(styles, { borderRadius: parseInt(radius.all || "0") }); diff --git a/packages/db-dynamodb/src/DynamoDbDriver.ts b/packages/db-dynamodb/src/DynamoDbDriver.ts index 45b5a13d2a5..7e3ccf28ec9 100644 --- a/packages/db-dynamodb/src/DynamoDbDriver.ts +++ b/packages/db-dynamodb/src/DynamoDbDriver.ts @@ -1,48 +1,19 @@ import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { DbDriver, Result } from "@webiny/db"; +import { DbDriver } from "@webiny/db"; interface ConstructorArgs { documentClient: DynamoDBDocument; } -class DynamoDbDriver implements DbDriver { - batchProcesses: Record; - documentClient: DynamoDBDocument; +class DynamoDbDriver implements DbDriver { + public readonly documentClient: DynamoDBDocument; constructor({ documentClient }: ConstructorArgs) { - this.batchProcesses = {}; this.documentClient = documentClient; } getClient() { return this.documentClient; } - async create(): Promise { - return [true, {}]; - } - - async update(): Promise { - return [true, {}]; - } - - async delete(): Promise { - return [true, {}]; - } - - async read(): Promise> { - return [[], {}]; - } - - async createLog(): Promise { - return [true, {}]; - } - - async readLogs() { - return this.read(); - } - - getBatchProcess() { - // not empty - } } export default DynamoDbDriver; diff --git a/packages/db-dynamodb/src/types.ts b/packages/db-dynamodb/src/types.ts index 7462e21df3a..83d721d7b99 100644 --- a/packages/db-dynamodb/src/types.ts +++ b/packages/db-dynamodb/src/types.ts @@ -1,27 +1,3 @@ -export interface OperatorArgs { - expression: string; - attributeNames: Record; - attributeValues: Record; -} - -interface CanProcessArgs { - key: string; - value: any; - args: OperatorArgs; -} - -interface ProcessArgs { - key: string; - value: any; - args: OperatorArgs; - processStatement: any; -} - -export interface Operator { - canProcess: ({ key }: CanProcessArgs) => boolean; - process: ({ key, value, args }: ProcessArgs) => void; -} - /** * We use this definition to search for a value in any given field that was passed. * It works as an "OR" condition. diff --git a/packages/db-dynamodb/src/utils/batchRead.ts b/packages/db-dynamodb/src/utils/batchRead.ts index 1324297674c..74a51e5a220 100644 --- a/packages/db-dynamodb/src/utils/batchRead.ts +++ b/packages/db-dynamodb/src/utils/batchRead.ts @@ -1,6 +1,7 @@ import lodashChunk from "lodash/chunk"; import WebinyError from "@webiny/error"; import { TableDef } from "~/toolbox"; +import { GenericRecord } from "@webiny/api/types"; export interface BatchReadItem { Table?: TableDef; @@ -57,7 +58,7 @@ const batchReadAllChunk = async (params: BatchReadAllChunkParams): Prom * This helper function is meant to be used to batch read from one table. * It will fetch all results, as there is a next() method call built in. */ -export const batchReadAll = async ( +export const batchReadAll = async ( params: BatchReadParams, maxChunk = MAX_BATCH_ITEMS ): Promise => { @@ -75,7 +76,7 @@ export const batchReadAll = async ( const records: T[] = []; - const chunkItemsList: BatchReadItem[][] = lodashChunk(params.items, maxChunk); + const chunkItemsList = lodashChunk(params.items, maxChunk); for (const chunkItems of chunkItemsList) { const results = await batchReadAllChunk({ diff --git a/packages/db/package.json b/packages/db/package.json index d1938043576..d42fc84139f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -12,6 +12,9 @@ "access": "public", "directory": "dist" }, + "dependencies": { + "@webiny/api": "0.0.0" + }, "devDependencies": { "@webiny/cli": "0.0.0", "@webiny/project-utils": "0.0.0", diff --git a/packages/db/src/DbRegistry.ts b/packages/db/src/DbRegistry.ts new file mode 100644 index 00000000000..3bbbf8fc7e1 --- /dev/null +++ b/packages/db/src/DbRegistry.ts @@ -0,0 +1,49 @@ +import { IRegistry, IRegistryItem, IRegistryRegisterParams } from "./types"; +import { GenericRecord } from "@webiny/api/types"; + +export class DbRegistry implements IRegistry { + private readonly items: GenericRecord = {}; + + public register(input: IRegistryRegisterParams): void { + const key = `${input.app}-${input.tags.sort().join("-")}`; + + if (this.items[key]) { + throw new Error( + `Item with app "${input.app}" and tags "${input.tags.join( + ", " + )}" is already registered.` + ); + } + this.items[key] = input; + } + + public getOneItem(cb: (item: IRegistryItem) => boolean): IRegistryItem { + const item = this.getItem(cb); + if (!item) { + throw new Error("Item not found."); + } + return item; + } + + public getItem(cb: (item: IRegistryItem) => boolean): IRegistryItem | null { + const items = this.getItems(cb); + if (items.length === 0) { + return null; + } else if (items.length > 1) { + throw new Error("More than one item found with the provided criteria."); + } + return items[0]; + } + + public getItems(cb: (item: IRegistryItem) => boolean): IRegistryItem[] { + const results: IRegistryItem[] = []; + for (const key in this.items) { + const item = this.items[key] as IRegistryItem; + if (cb(item)) { + results.push(item); + } + } + + return results; + } +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index e06d9da2d2b..01a56af9175 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,254 +1,26 @@ -/** - * TODO Remove when moved all packages to standalone storage opts. - */ -interface KeyField { - name: string; -} +import { DbRegistry } from "~/DbRegistry"; -export interface Key { - primary?: boolean; - unique?: boolean; - name: string; - fields: KeyField[]; -} +export * from "./types"; -export interface ArgsBatch { - instance: Batch; - operation: Operation; -} -export interface Args { - __batch?: ArgsBatch; - table?: string; - meta?: boolean; - limit?: number; - sort?: Record; - data?: Record; - query?: Record; - keys?: Key[]; +export interface DbDriver { + getClient(): T; } -export type Result = [T, Record]; - -export interface DbDriver { - create: (args: Args) => Promise>; - read: >(args: Args) => Promise>; - update: (args: Args) => Promise>; - delete: (args: Args) => Promise>; - - // Logging functions. - createLog: (args: { - operation: string; - data: Args; - table: string; - id: string; - }) => Promise>; - readLogs: >(args: { table: string }) => Promise>; -} - -export type OperationType = "create" | "read" | "update" | "delete"; -export type Operation = [OperationType, Args]; - -export type ConstructorArgs = { - driver: DbDriver; +export interface ConstructorArgs { + driver: DbDriver; table?: string; - logTable?: string; -}; - -// Generates a short and sortable ID, e.g. "1607677774994.tfz58m". -const shortId = () => { - const time = new Date().getTime(); - const uniqueId = Math.random().toString(36).slice(-6); - - return `${time}.${uniqueId}`; -}; - -interface LogDataBatch { - id: string; - type: string; -} -interface LogData - extends Pick { - batch: LogDataBatch | null; } -// Picks necessary data from received args, ready to be stored in the log table. -const getCreateLogData = (args: Args): LogData => { - const { table, meta, limit, sort, data, query, keys } = args; +class Db { + public driver: DbDriver; + public readonly table?: string; - return { - table, - meta, - limit, - sort, - data, - query, - keys, - batch: args.__batch - ? { - id: args.__batch.instance.id, - type: args.__batch.instance.type - } - : null - }; -}; + public readonly registry = new DbRegistry(); -class Db { - public driver: DbDriver; - public table: string; - public logTable?: string; - - constructor({ driver, table, logTable }: ConstructorArgs) { - this.driver = driver; - // @ts-expect-error + constructor({ driver, table }: ConstructorArgs) { this.table = table; - this.logTable = logTable; - } - - public async create(args: Args): Promise> { - const createArgs = { ...args, table: args.table || this.table }; - await this.createLog("create", createArgs); - return this.driver.create(createArgs); - } - - public async read>(args: Args): Promise> { - const readArgs = { ...args, table: args.table || this.table }; - await this.createLog("read", readArgs); - return this.driver.read(readArgs); - } - - public async update(args: Args): Promise> { - const updateArgs = { ...args, table: args.table || this.table }; - await this.createLog("update", updateArgs); - return this.driver.update(updateArgs); - } - - public async delete(args: Args): Promise> { - const deleteArgs = { ...args, table: args.table || this.table }; - await this.createLog("delete", deleteArgs); - return this.driver.delete(deleteArgs); - } - - // Logging functions. - public async createLog(operation: string, args: Args): Promise | null> { - if (!this.logTable) { - return null; - } - - const data = getCreateLogData(args); - return this.driver.createLog({ operation, data, table: this.logTable, id: shortId() }); - } - - public async readLogs>(): Promise | null> { - if (!this.logTable) { - return null; - } - - return this.driver.readLogs({ - table: this.logTable - }); - } - - public batch< - T0 = any, - T1 = any, - T2 = any, - T3 = any, - T4 = any, - T5 = any, - T6 = any, - T7 = any, - T8 = any, - T9 = any - >(): Batch { - return new Batch(this); - } -} - -class Batch< - T0 = any, - T1 = any, - T2 = any, - T3 = any, - T4 = any, - T5 = any, - T6 = any, - T7 = any, - T8 = any, - T9 = any -> { - db: Db; - type: "batch" | "transaction"; - id: string; - meta: Record; - operations: Operation[]; - - constructor(db: Db) { - this.db = db; - this.type = "batch"; - this.id = shortId(); - - this.meta = {}; - this.operations = []; - } - - push(...operations: Operation[]) { - for (let i = 0; i < operations.length; i++) { - const item = operations[i]; - this.operations.push(item); - } - return this; - } - - create(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["create", args[i]]); - } - return this; - } - - read(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["read", args[i]]); - } - return this; - } - - update(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["update", args[i]]); - } - return this; - } - - delete(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["delete", args[i]]); - } - return this; - } - - async execute(): Promise<[T0?, T1?, T2?, T3?, T4?, T5?, T6?, T7?, T8?, T9?]> { - /** - * TODO: figure out which exact type to use instead of any. - */ - const promises: Promise[] = []; - for (let i = 0; i < this.operations.length; i++) { - const [operation, args] = this.operations[i]; - promises.push( - this.db[operation]({ - ...args, - __batch: { - instance: this, - operation: this.operations[i] - } - }) - ); - } - - const result = Promise.all(promises); - - return result as Promise<[T0?, T1?, T2?, T3?, T4?, T5?, T6?, T7?, T8?, T9?]>; + this.driver = driver; } } -export { Batch, Db }; +export { Db }; diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts new file mode 100644 index 00000000000..eac650c993b --- /dev/null +++ b/packages/db/src/types.ts @@ -0,0 +1,26 @@ +import { NonEmptyArray } from "@webiny/api/types"; + +export interface IRegistryRegisterParams { + item: T; + app: string; + tags: NonEmptyArray; +} + +export interface IRegistryItem { + item: T; + app: string; + tags: NonEmptyArray; +} + +export interface IRegistry { + register(params: IRegistryRegisterParams): void; + /** + * Throws an error if more than one item is found or there is no item found. + */ + getOneItem(cb: (item: IRegistryItem) => boolean): IRegistryItem; + /** + * Throws an error if more than one item is found. + */ + getItem(cb: (item: IRegistryItem) => boolean): IRegistryItem | null; + getItems(cb: (item: IRegistryItem) => boolean): IRegistryItem[]; +} diff --git a/packages/db/tsconfig.build.json b/packages/db/tsconfig.build.json index 5e7843d3c8c..097b4c0b400 100644 --- a/packages/db/tsconfig.build.json +++ b/packages/db/tsconfig.build.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.build.json", "include": ["src"], - "references": [], + "references": [{ "path": "../api/tsconfig.build.json" }], "compilerOptions": { "rootDir": "./src", "outDir": "./dist", diff --git a/packages/db/tsconfig.json b/packages/db/tsconfig.json index 6ca26f3c929..306a82a42a7 100644 --- a/packages/db/tsconfig.json +++ b/packages/db/tsconfig.json @@ -1,12 +1,17 @@ { "extends": "../../tsconfig.json", "include": ["src", "__tests__"], - "references": [], + "references": [{ "path": "../api" }], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], "outDir": "./dist", "declarationDir": "./dist", - "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, + "paths": { + "~/*": ["./src/*"], + "~tests/*": ["./__tests__/*"], + "@webiny/api/*": ["../api/src/*"], + "@webiny/api": ["../api/src"] + }, "baseUrl": "." } } diff --git a/packages/handler-db/src/index.ts b/packages/handler-db/src/index.ts index af1cbeb82b5..233c7e387a2 100644 --- a/packages/handler-db/src/index.ts +++ b/packages/handler-db/src/index.ts @@ -1,17 +1,14 @@ -import { Db } from "@webiny/db"; +import { ConstructorArgs, Db } from "@webiny/db"; import { ContextPlugin } from "@webiny/api"; import { DbContext } from "./types"; -/** - * TODO: remove this package. - */ -export default (args: any) => { - return [ - new ContextPlugin(context => { - if (context.db) { - return; - } - context.db = new Db(args); - }) - ]; +export default (args: ConstructorArgs) => { + const plugin = new ContextPlugin(context => { + if (context.db) { + return; + } + context.db = new Db(args); + }); + plugin.name = "handler-db.context.db"; + return [plugin]; }; diff --git a/packages/handler-db/src/types.ts b/packages/handler-db/src/types.ts index 6937b8bd4ca..bfa9b7fb0e6 100644 --- a/packages/handler-db/src/types.ts +++ b/packages/handler-db/src/types.ts @@ -2,5 +2,5 @@ import { Db } from "@webiny/db"; import { Context } from "@webiny/api/types"; export interface DbContext extends Context { - db: Db; + db: Db; } diff --git a/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts b/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts index a20a637ac92..ef5fb822c31 100644 --- a/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts +++ b/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts @@ -4,12 +4,14 @@ import { GraphQLSchemaDefinition, ResolverDecorators, Resolvers, TypeDefs } from export interface IGraphQLSchemaPlugin extends Plugin { schema: GraphQLSchemaDefinition; + isApplicable: (context: TContext) => boolean; } export interface GraphQLSchemaPluginConfig { typeDefs?: TypeDefs; resolvers?: Resolvers; resolverDecorators?: ResolverDecorators; + isApplicable?: (context: TContext) => boolean; } export class GraphQLSchemaPlugin @@ -31,6 +33,13 @@ export class GraphQLSchemaPlugin resolverDecorators: this.config.resolverDecorators }; } + + isApplicable(context: TContext): boolean { + if (this.config.isApplicable) { + return this.config.isApplicable(context); + } + return true; + } } export const createGraphQLSchemaPlugin = (config: GraphQLSchemaPluginConfig) => { diff --git a/packages/tasks/src/runner/TaskManager.ts b/packages/tasks/src/runner/TaskManager.ts index 0cb6732b7ed..e9ee12c642b 100644 --- a/packages/tasks/src/runner/TaskManager.ts +++ b/packages/tasks/src/runner/TaskManager.ts @@ -16,15 +16,17 @@ import { } from "~/response/abstractions"; import { getErrorProperties } from "~/utils/getErrorProperties"; +type ITaskManagerRunner = Pick; + export class TaskManager implements ITaskManager { - private readonly runner: Pick; + private readonly runner: ITaskManagerRunner; private readonly context: Context; private readonly response: IResponse; private readonly taskResponse: ITaskResponse; private readonly store: ITaskManagerStorePrivate; public constructor( - runner: Pick, + runner: ITaskManagerRunner, context: Context, response: IResponse, taskResponse: ITaskResponse, @@ -134,7 +136,8 @@ export class TaskManager implements ITaskManager { ...params, parent: this.store.getTask() }); - } + }, + timer: this.runner.timer }); }); } catch (ex) { diff --git a/packages/tasks/src/runner/TaskRunner.ts b/packages/tasks/src/runner/TaskRunner.ts index 13db64c6741..f758aecf108 100644 --- a/packages/tasks/src/runner/TaskRunner.ts +++ b/packages/tasks/src/runner/TaskRunner.ts @@ -23,8 +23,8 @@ export class TaskRunner implements ITaskRunner { * Follow the same example for the rest of the properties. */ public readonly context: C; + public readonly timer: ITimer; private readonly validation: ITaskEventValidation; - private readonly timer: ITimer; /** * We take all required variables separately because they will get injected via DI - so less refactoring is required in the future. diff --git a/packages/tasks/src/runner/abstractions/TaskRunner.ts b/packages/tasks/src/runner/abstractions/TaskRunner.ts index da4f8bca9fb..3d399826041 100644 --- a/packages/tasks/src/runner/abstractions/TaskRunner.ts +++ b/packages/tasks/src/runner/abstractions/TaskRunner.ts @@ -1,6 +1,7 @@ import { Context } from "~/types"; import { ITaskEvent } from "~/handler/types"; import { IResponseResult } from "~/response/abstractions"; +import { ITimer } from "@webiny/handler-aws"; export interface IIsCloseToTimeoutCallable { (seconds?: number): boolean; @@ -9,5 +10,6 @@ export interface IIsCloseToTimeoutCallable { export interface ITaskRunner { context: C; isCloseToTimeout: IIsCloseToTimeoutCallable; + timer: ITimer; run(event: ITaskEvent): Promise; } diff --git a/packages/tasks/src/types.ts b/packages/tasks/src/types.ts index e3619ca6d8d..b7447934182 100644 --- a/packages/tasks/src/types.ts +++ b/packages/tasks/src/types.ts @@ -17,6 +17,7 @@ import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "./runner/abstracti import { SecurityPermission } from "@webiny/api-security/types"; import { GenericRecord } from "@webiny/api/types"; import { IStepFunctionServiceFetchResult } from "~/service/StepFunctionServicePlugin"; +import { ITimer } from "@webiny/handler-aws"; import type zod from "zod"; @@ -334,6 +335,7 @@ export interface ITaskRunParams< trigger( params: Omit, "parent"> ): Promise>; + timer: ITimer; } export interface ITaskOnSuccessParams< diff --git a/yarn.lock b/yarn.lock index b8ce65e37dd..f2017a53648 100644 --- a/yarn.lock +++ b/yarn.lock @@ -282,24 +282,24 @@ __metadata: linkType: hard "@aws-sdk/client-apigatewaymanagementapi@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-apigatewaymanagementapi@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-apigatewaymanagementapi@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -326,29 +326,29 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 9c4de371e631de9e21f6e89b9354bd7bedfc094ce79b3e3b1229932ce1b974c6e7691ae1df62250bdc3d0a62363b6e1047ffd9b1649f7e59b9a8d130f422a4db + checksum: 7d57c1afd6ba8c3de634109636ea5553cde60431f396a8417b526462bb1c18b154d34360b95d6a87af4601422c3279e7158e56ad1733d00f447d783201804c18 languageName: node linkType: hard "@aws-sdk/client-cloudfront@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-cloudfront@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-cloudfront@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@aws-sdk/xml-builder": 3.662.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 @@ -378,29 +378,29 @@ __metadata: "@smithy/util-utf8": ^3.0.0 "@smithy/util-waiter": ^3.1.6 tslib: ^2.6.2 - checksum: d7ba82fdc1f5ba035481caf0c455ed58ad3ba35e64ebfae7d2d1cf1033b93b42ea385388a0a43ae84f867e5ee7082f3f5ebb0884401cff2abe837b03c20c421e + checksum: 85f64810b8227fead98aa98100d7bdc0b538f76a97111b9df99e3464d407caa94a3f05d1ff54f2ff012e44392fbb5476916f304052dff917c7f3084158b714de languageName: node linkType: hard "@aws-sdk/client-cloudwatch-events@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-cloudwatch-events@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-cloudwatch-events@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -427,7 +427,7 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 91fd5282340dbc0e7c3e4633284b7fe5a062c1744aa407403ba6702502aba1af14faf68aaf1a791386cd27564c0d538b74721eb1fd4854c0cdc67c253e82ad11 + checksum: 9071d178a6923340025537e8ef8fffbeaa9f44bf980ff815efff032ad5fd48bdb10f8ae14e883028a97e12965720b8044cddd052884539a78511730e00323d0f languageName: node linkType: hard @@ -471,24 +471,24 @@ __metadata: linkType: hard "@aws-sdk/client-cloudwatch-logs@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-cloudwatch-logs@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-cloudwatch-logs@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/eventstream-serde-browser": ^3.0.10 @@ -519,29 +519,29 @@ __metadata: "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 uuid: ^9.0.1 - checksum: 03015d6d53b0dc1561d3f7a5507f9132c13384a6b08d52f59601e810fe1fc86ad7e393d74f42b92eddce06d08c63b3cce075b403298880b09e70fd2592778b39 + checksum: 33ad4a03500b7ab2b5e71941c7517e045e99f360e5cc6f620a6d12bedd0ddc0cb7bb48b76fc2847917170f3fc906876a2e2066f92060b6cd5a8637d9e28e07a4 languageName: node linkType: hard "@aws-sdk/client-cognito-identity-provider@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-cognito-identity-provider@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-cognito-identity-provider@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -568,7 +568,7 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: a532ea5bfa6db3689be7f1f52c516e00096111dffa96904a88c2db4a9055ffb401ebd56aaa1e40bf07a72be445c7cffaba63703a6c31edc98ced68d521dc166c + checksum: 525520a8fc228ca7a6eb8549d71a2ee910e78bef6cbd20ff12626f7f8535f5ab4b6a94e0279e13d13a575dff6b539158fd5fc69b726eb77bba8aa323e3534492 languageName: node linkType: hard @@ -611,25 +611,25 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-cognito-identity@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/client-cognito-identity@npm:3.668.0" +"@aws-sdk/client-cognito-identity@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/client-cognito-identity@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -656,29 +656,29 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: f6c96aae11c1885f1d0cb526ad9c5914396b988a9a3b15f9147980f3b28c4a0d5f17519b5d8032566ac51331ac738bb6da6ca4ed723588ef5f4edf21ad2c38f9 + checksum: 44c6615ad0fe81e63bb7c4f9c7f74280dbb4500215c77f8cdb3c537799977a1ef7ecee1c059cd2df3660b07669200bbcde8b97c7b7cfb2fa30e0482d88267ee3 languageName: node linkType: hard "@aws-sdk/client-dynamodb-streams@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-dynamodb-streams@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-dynamodb-streams@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -705,30 +705,30 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 9574a89d16825137fcd5034518f20b2f0f3cb8c75532bd8330a6b7f1b727aa79aae542086ee9a4d7916403ac233741118f2d44353e6b663bfa0152dc03b74b93 + checksum: da6dd3d50feabeffd8d7f14b672585dab05b0591a875382b441d7046f8522c12954e4a4480720b97890be8909a612744a4e238a3034184453f916c7f1fa4650a languageName: node linkType: hard "@aws-sdk/client-dynamodb@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-dynamodb@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-endpoint-discovery": 3.667.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -757,30 +757,30 @@ __metadata: "@smithy/util-waiter": ^3.1.6 tslib: ^2.6.2 uuid: ^9.0.1 - checksum: 888459faece6cce53248bc82c67e9cbba6cc2b7ab2dd5689fa21961e3f57b0aa89fa109ee6fb29845cc3fdc39f1c6a5f08bca18c0d9d8341eba3820ca76a83ea + checksum: e9a139d4542ef73845d86927a2e8a1b2624aa7e13142864dd217392cb55f8b19a063dc1c08e541787be578f4ee8f0af6bb2acaff06e4083c710b72cf71fcd44a languageName: node linkType: hard "@aws-sdk/client-eventbridge@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-eventbridge@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-eventbridge@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 - "@aws-sdk/signature-v4-multi-region": 3.667.0 + "@aws-sdk/signature-v4-multi-region": 3.669.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -807,29 +807,29 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 64a6c4262d3c23325074c97618bda898273f6d6d6d8bde21b292bc8278ae4368ee988316289424d37c77c7fcda998770899c8cc278644f5a5e2d231e7bb450a1 + checksum: 3c628c0bfa94e745a08adaea0980fd814433d6e792f2eb71f6a96cb2f5d19b09c07364212f72cdcb1069d147f5d8c6e5b7d29491b5e42b6ff0bcfd450b908bf6 languageName: node linkType: hard "@aws-sdk/client-iam@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-iam@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-iam@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -857,29 +857,29 @@ __metadata: "@smithy/util-utf8": ^3.0.0 "@smithy/util-waiter": ^3.1.6 tslib: ^2.6.2 - checksum: 656702a93f719e8ffdea68664cd45c7e95dc4afb42073b1b8a21ca73d8c1db39b73c9efe521f6ac58fc34542e33adf2eb3abb916becfe722d7f34f7816dbe4a2 + checksum: ab67fd3c0da9651f7af0f9bd58819d5a344dc61d8167a01c2f37850165760d5b474f00eb1c668cda7ad1cee4371cf8782d09c6400a99975786628515d4e6e298 languageName: node linkType: hard "@aws-sdk/client-iot@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-iot@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-iot@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -907,29 +907,29 @@ __metadata: "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 uuid: ^9.0.1 - checksum: 3f55dc542f35f4434d52f28bdc920c0c79218bec1f8e759f3d18f73505732a68adc9ae31ea7882b67c2d6092a00970b2f051a546ce7a6f580dc1165943751ac2 + checksum: 16f77da891fd93f3a000bff8825499260f3d28cff5589e9b1aca970342c969bbafcdd6d597ce68df2e8de32a17e76b5a0f3d9cabe4a670eaefe7f695773c1626 languageName: node linkType: hard "@aws-sdk/client-lambda@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-lambda@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-lambda@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/eventstream-serde-browser": ^3.0.10 @@ -961,37 +961,37 @@ __metadata: "@smithy/util-utf8": ^3.0.0 "@smithy/util-waiter": ^3.1.6 tslib: ^2.6.2 - checksum: 8e10395aa975189f44005478b301f76137153d3cf9abf400ffef26ba9265c1b0bdb35c442728d97c17898cc54544243d2f7e16c32ee850afa122ad741298e3b7 + checksum: 49f51fdb12e6423c9f9ac4a6d78316839022f962b4ef55f259fd3b23fbbb7e3b032319b63703b1a48e18f9793fbf0394b62c46bda82a7b3f4149ba9d1f16f70f languageName: node linkType: hard -"@aws-sdk/client-s3@npm:3.668.0, @aws-sdk/client-s3@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-s3@npm:3.668.0" +"@aws-sdk/client-s3@npm:3.670.0, @aws-sdk/client-s3@npm:^3.654.0": + version: 3.670.0 + resolution: "@aws-sdk/client-s3@npm:3.670.0" dependencies: "@aws-crypto/sha1-browser": 5.2.0 "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-bucket-endpoint": 3.667.0 "@aws-sdk/middleware-expect-continue": 3.667.0 - "@aws-sdk/middleware-flexible-checksums": 3.667.0 + "@aws-sdk/middleware-flexible-checksums": 3.669.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-location-constraint": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-sdk-s3": 3.667.0 + "@aws-sdk/middleware-sdk-s3": 3.669.0 "@aws-sdk/middleware-ssec": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 - "@aws-sdk/signature-v4-multi-region": 3.667.0 + "@aws-sdk/signature-v4-multi-region": 3.669.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@aws-sdk/xml-builder": 3.662.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 @@ -1027,29 +1027,29 @@ __metadata: "@smithy/util-utf8": ^3.0.0 "@smithy/util-waiter": ^3.1.6 tslib: ^2.6.2 - checksum: 8de4bf7973657cc8e98b854971b59f47003fe854ad6d49ab3a97ac4dacb95f830a615aac43e11cc9e6e1536227325ca796a3f3e4a9cf59e33be8a26592957d01 + checksum: 64e78b5d1ccd196f38f1b7d074eb21f90ebb151e5e92317afee66c654c73717ff70bccfe20322942d76a514d517d112a7595f3bda4e6dd22ffcf42fe62d9f3f4 languageName: node linkType: hard "@aws-sdk/client-sfn@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-sfn@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-sfn@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -1077,30 +1077,30 @@ __metadata: "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 uuid: ^9.0.1 - checksum: 0bb0f8e1da404b956838005ece470f7fe76a0d1896003c40abad0d8117a7b77efad2dce5ba098a308275f3cce729589deed7578249fc20343f5940d88fd96a2a + checksum: 37d23f718471393c66555ec6598211778d8eae28f2c03d074678ead35107116c0d25ba35be9b21745dde0efadf1110614669746f0b190e2741ed94b3819d715b languageName: node linkType: hard "@aws-sdk/client-sqs@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-sqs@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/client-sqs@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 "@aws-sdk/middleware-sdk-sqs": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -1128,27 +1128,27 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 27979a2b2775096da06907f8eece31ec89e8a0b1bbaeae5d00380196058f2c5a4c3d0f7656d81bdbef523e4cfc884e31c4a13f3bbde7a38a7d722086ba8c0340 + checksum: 7f5435a328605031ee023df47f81822db7a0e14de49e26109ada475759865de48c5237b5a6e37cf7f4c24a350401a78e0a1a922aaa3849affa60ae3667f0db12 languageName: node linkType: hard -"@aws-sdk/client-sso-oidc@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/client-sso-oidc@npm:3.668.0" +"@aws-sdk/client-sso-oidc@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/client-sso-oidc@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -1176,14 +1176,14 @@ __metadata: "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 peerDependencies: - "@aws-sdk/client-sts": ^3.668.0 - checksum: cc74edf7b530183f48a93ecc8ca1d8dad48669da4117a78c98906dd6b5791f7f8f2dad3699a80ab9d0cbe60c56758399d858b31d6f5d98c37e216ded468ff810 + "@aws-sdk/client-sts": ^3.670.0 + checksum: 3f3f10daab3629f6159c200abe175f147ebefe3e89acff83b31494674ac693c44043fcea87529258512e01e1bee4785a3d6861f50e05b166027d1f185f0dca54 languageName: node linkType: hard -"@aws-sdk/client-sso@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/client-sso@npm:3.668.0" +"@aws-sdk/client-sso@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/client-sso@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 @@ -1191,12 +1191,12 @@ __metadata: "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -1223,28 +1223,28 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: ed4df5a48b1465b24832897314fc6ba0f1d433de8c7e36ef77f3a445fc55d29d01020a4f2e4489f16dbd26ed0982261bb2545ec142d3393a5ec36ebdcea45add + checksum: 226bb40e8b38e60f016bedc6586e28f487e872f6e128bdd32f4b5443431b4e5e7a05a0b1b7f8d8eb309b02f9eca9ec0606cedcdbe585a189e3b1d6d633f0e49b languageName: node linkType: hard -"@aws-sdk/client-sts@npm:3.668.0, @aws-sdk/client-sts@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/client-sts@npm:3.668.0" +"@aws-sdk/client-sts@npm:3.670.0, @aws-sdk/client-sts@npm:^3.654.0": + version: 3.670.0 + resolution: "@aws-sdk/client-sts@npm:3.670.0" dependencies: "@aws-crypto/sha256-browser": 5.2.0 "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.668.0 + "@aws-sdk/client-sso-oidc": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/middleware-host-header": 3.667.0 "@aws-sdk/middleware-logger": 3.667.0 "@aws-sdk/middleware-recursion-detection": 3.667.0 - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/region-config-resolver": 3.667.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-endpoints": 3.667.0 - "@aws-sdk/util-user-agent-browser": 3.667.0 - "@aws-sdk/util-user-agent-node": 3.668.0 + "@aws-sdk/util-user-agent-browser": 3.670.0 + "@aws-sdk/util-user-agent-node": 3.669.0 "@smithy/config-resolver": ^3.0.9 "@smithy/core": ^2.4.8 "@smithy/fetch-http-handler": ^3.2.9 @@ -1271,7 +1271,7 @@ __metadata: "@smithy/util-retry": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: dae0500287ae7a83ce98e06ec904914ab2c9c51b948d0111b2921ff3416b3035ec12ca576c10fa1cc46d73b5fe23ad32277579c64f20ff1a10b3cfec4cb5b033 + checksum: 3cd0a8a5db4fe9596ed89e84acc75dcfa662c3d76dc171bfdff9ba35b5e5c67bafad1391fb9167a3533eec46933c6e4bbc723096ffee1243cbcb177ba2538dc2 languageName: node linkType: hard @@ -1317,16 +1317,16 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-cognito-identity@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/credential-provider-cognito-identity@npm:3.668.0" +"@aws-sdk/credential-provider-cognito-identity@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/credential-provider-cognito-identity@npm:3.670.0" dependencies: - "@aws-sdk/client-cognito-identity": 3.668.0 + "@aws-sdk/client-cognito-identity": 3.670.0 "@aws-sdk/types": 3.667.0 "@smithy/property-provider": ^3.1.7 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: 7b73a2bd0c460d4cee823509f4653f81b20ba4b5dfd967f930e7e39720b0247086f045ca2bc8533970627092ada5ed37a6566ddce96a4313db9750cec101cd64 + checksum: 0bd6dec06536b2be919d1fdaf526fbe89a7a7bc3a200e38133a76fef5af2a70883e6c3309842ca511021c247fdf4604612ade189d7ccd5de8214cc0db8f2f4d7 languageName: node linkType: hard @@ -1395,15 +1395,15 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-ini@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/credential-provider-ini@npm:3.668.0" +"@aws-sdk/credential-provider-ini@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/credential-provider-ini@npm:3.670.0" dependencies: "@aws-sdk/core": 3.667.0 "@aws-sdk/credential-provider-env": 3.667.0 "@aws-sdk/credential-provider-http": 3.667.0 "@aws-sdk/credential-provider-process": 3.667.0 - "@aws-sdk/credential-provider-sso": 3.668.0 + "@aws-sdk/credential-provider-sso": 3.670.0 "@aws-sdk/credential-provider-web-identity": 3.667.0 "@aws-sdk/types": 3.667.0 "@smithy/credential-provider-imds": ^3.2.4 @@ -1412,8 +1412,8 @@ __metadata: "@smithy/types": ^3.5.0 tslib: ^2.6.2 peerDependencies: - "@aws-sdk/client-sts": ^3.668.0 - checksum: 4faf14681971872d5a8400e4a518aaa8e4659e54c6adec4cfc0c59a30448061ba95b6d8b120d8535aecf11a8b4bbccde6bf5dbb5a16ff927f287f3c14214ba65 + "@aws-sdk/client-sts": ^3.670.0 + checksum: 63afbf4a8d571c2f029efbe7bbc2c81089b250aad0a63aa4f8cccda08650f686795e90f449e25c8b17739c8631fa11d690f6a54b38f174340bb45a1e57e187b8 languageName: node linkType: hard @@ -1433,15 +1433,15 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-node@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/credential-provider-node@npm:3.668.0" +"@aws-sdk/credential-provider-node@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/credential-provider-node@npm:3.670.0" dependencies: "@aws-sdk/credential-provider-env": 3.667.0 "@aws-sdk/credential-provider-http": 3.667.0 - "@aws-sdk/credential-provider-ini": 3.668.0 + "@aws-sdk/credential-provider-ini": 3.670.0 "@aws-sdk/credential-provider-process": 3.667.0 - "@aws-sdk/credential-provider-sso": 3.668.0 + "@aws-sdk/credential-provider-sso": 3.670.0 "@aws-sdk/credential-provider-web-identity": 3.667.0 "@aws-sdk/types": 3.667.0 "@smithy/credential-provider-imds": ^3.2.4 @@ -1449,7 +1449,7 @@ __metadata: "@smithy/shared-ini-file-loader": ^3.1.8 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: 31b2e9e4e256feab3717a1be53d179389bef5229aac952bd21af3e681095d3cb743b5b06f0594a0dc363d8c790854152c9bbf66ece83f83a7f4d7ba0c914a071 + checksum: b25c6ca674f08905b62a14818e4d136beb2f3c96ac84cb8be6885e30c07b60026dc9bab2a57d509f5a7e9da234ea914c979f6a64e01b21d399c0f1a4eab55fa0 languageName: node linkType: hard @@ -1480,11 +1480,11 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-sso@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/credential-provider-sso@npm:3.668.0" +"@aws-sdk/credential-provider-sso@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/credential-provider-sso@npm:3.670.0" dependencies: - "@aws-sdk/client-sso": 3.668.0 + "@aws-sdk/client-sso": 3.670.0 "@aws-sdk/core": 3.667.0 "@aws-sdk/token-providers": 3.667.0 "@aws-sdk/types": 3.667.0 @@ -1492,7 +1492,7 @@ __metadata: "@smithy/shared-ini-file-loader": ^3.1.8 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: b84ebe583d68e647030b16460f2576b1f498c7c9f7b1f8bee2011574dc239b570fc79df73ebb92f2d55f174efe732a1b39b8326990fb83ac0e463b7fb9e2352a + checksum: cb86ce737aa72fe596d523c2d0e80f491857b2f794f046389f9b193c52bc7ecac8a91b4c92bb8ad3f8eda35a071db0b5564c4bbd65aa5dd945c62462c89e7197 languageName: node linkType: hard @@ -1512,27 +1512,27 @@ __metadata: linkType: hard "@aws-sdk/credential-providers@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/credential-providers@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/credential-providers@npm:3.670.0" dependencies: - "@aws-sdk/client-cognito-identity": 3.668.0 - "@aws-sdk/client-sso": 3.668.0 - "@aws-sdk/client-sts": 3.668.0 + "@aws-sdk/client-cognito-identity": 3.670.0 + "@aws-sdk/client-sso": 3.670.0 + "@aws-sdk/client-sts": 3.670.0 "@aws-sdk/core": 3.667.0 - "@aws-sdk/credential-provider-cognito-identity": 3.668.0 + "@aws-sdk/credential-provider-cognito-identity": 3.670.0 "@aws-sdk/credential-provider-env": 3.667.0 "@aws-sdk/credential-provider-http": 3.667.0 - "@aws-sdk/credential-provider-ini": 3.668.0 - "@aws-sdk/credential-provider-node": 3.668.0 + "@aws-sdk/credential-provider-ini": 3.670.0 + "@aws-sdk/credential-provider-node": 3.670.0 "@aws-sdk/credential-provider-process": 3.667.0 - "@aws-sdk/credential-provider-sso": 3.668.0 + "@aws-sdk/credential-provider-sso": 3.670.0 "@aws-sdk/credential-provider-web-identity": 3.667.0 "@aws-sdk/types": 3.667.0 "@smithy/credential-provider-imds": ^3.2.4 "@smithy/property-provider": ^3.1.7 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: c29a09b6e465cd150a394a7e8bf6bd25cb49a40641614e85bfe2423d2ac6faf8f10a09df39a480475e70a9cf54dfe7fa652c601c507414dc9afb9e84d88ddd28 + checksum: ad413fefe856c4730bfd0be6fd664b84fdcfc441789f75f6a75951e0e2e4c390646b738b8b9537fff8b15270427b58259e3129324cd9cbbc11da2710be4880c4 languageName: node linkType: hard @@ -1590,23 +1590,24 @@ __metadata: linkType: hard "@aws-sdk/lib-dynamodb@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/lib-dynamodb@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/lib-dynamodb@npm:3.670.0" dependencies: - "@aws-sdk/util-dynamodb": 3.668.0 + "@aws-sdk/core": 3.667.0 + "@aws-sdk/util-dynamodb": 3.670.0 "@smithy/core": ^2.4.8 "@smithy/smithy-client": ^3.4.0 "@smithy/types": ^3.5.0 tslib: ^2.6.2 peerDependencies: - "@aws-sdk/client-dynamodb": ^3.668.0 - checksum: 83767234e14fc40f3be1db4729b383bbc7b4be230744cb4f9a021110cd89445fbbb435bcfe6819ca58aeddb85c3b17ce9340893d79c0225c1fb85a4f7fd4431e + "@aws-sdk/client-dynamodb": ^3.670.0 + checksum: 8c9246e74b56b4369e852c3568f5a377b813dd0fdfa017a30c12155af453afe63c2cd6aa1a0ef91aa2e8ebad96ff41a919f41299872060c2d54f150c41fac7b6 languageName: node linkType: hard "@aws-sdk/lib-storage@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/lib-storage@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/lib-storage@npm:3.670.0" dependencies: "@smithy/abort-controller": ^3.1.5 "@smithy/middleware-endpoint": ^3.1.4 @@ -1616,8 +1617,8 @@ __metadata: stream-browserify: 3.0.0 tslib: ^2.6.2 peerDependencies: - "@aws-sdk/client-s3": ^3.668.0 - checksum: 7261a15668dc1862fd8ff7390e540376a3812a302d831b11cce90f541c086e67b9291f1f5c3aa61927ab44dd15c43c5edc0ef15bcc10981c9ce51de6d0305913 + "@aws-sdk/client-s3": ^3.670.0 + checksum: 3dd6e6a9578ef6fbc024c1714268bca185a7d4b1d9e419a2bb627691659989731b1c4f0b74b575147aaea752eb0e057d17191f8d6e8f128e07ab0b829ae88df3 languageName: node linkType: hard @@ -1673,9 +1674,9 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/middleware-flexible-checksums@npm:3.667.0": - version: 3.667.0 - resolution: "@aws-sdk/middleware-flexible-checksums@npm:3.667.0" +"@aws-sdk/middleware-flexible-checksums@npm:3.669.0": + version: 3.669.0 + resolution: "@aws-sdk/middleware-flexible-checksums@npm:3.669.0" dependencies: "@aws-crypto/crc32": 5.2.0 "@aws-crypto/crc32c": 5.2.0 @@ -1688,7 +1689,7 @@ __metadata: "@smithy/util-middleware": ^3.0.7 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 9a78dae8daf15795e868584df362429a10bd29ad99b01d03a4256c6edfd10e1e2fbf6e236b53d1e9c7fd506774c8bd12dd5caf8b6cf8e5fe482321ca1e28d56e + checksum: 65fc21716ec7d1ff1f241e1fe90a1ae185a8e2421ebaa8b41118006b787df91335b1a6188bd4c7efa722ca427d71fae9439fbbe9d12fc9896ffba70a15540b7d languageName: node linkType: hard @@ -1773,9 +1774,9 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/middleware-sdk-s3@npm:3.667.0": - version: 3.667.0 - resolution: "@aws-sdk/middleware-sdk-s3@npm:3.667.0" +"@aws-sdk/middleware-sdk-s3@npm:3.669.0": + version: 3.669.0 + resolution: "@aws-sdk/middleware-sdk-s3@npm:3.669.0" dependencies: "@aws-sdk/core": 3.667.0 "@aws-sdk/types": 3.667.0 @@ -1791,7 +1792,7 @@ __metadata: "@smithy/util-stream": ^3.1.9 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 80407de1e6a356425ced1fceeb8235107464e45add95615d9049590d413e5b483d4a3a37e6cd9e75524b952dd825a2c43daf3bb06c06b5eb989868c9707a11df + checksum: 9137a0d67091ea9fa134a90cff8e5a1b1f38fd817cb2cd419cb0a006f9ad52e0fa74a94f56c1f1963dc8faa928b16ce8e237af4994bcc5da1d22ec25017e15b1 languageName: node linkType: hard @@ -1862,9 +1863,9 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/middleware-user-agent@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/middleware-user-agent@npm:3.668.0" +"@aws-sdk/middleware-user-agent@npm:3.669.0": + version: 3.669.0 + resolution: "@aws-sdk/middleware-user-agent@npm:3.669.0" dependencies: "@aws-sdk/core": 3.667.0 "@aws-sdk/types": 3.667.0 @@ -1873,7 +1874,7 @@ __metadata: "@smithy/protocol-http": ^4.1.4 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: 43ab68954efaf3f742b6d0df747e13f4356661fb2b15a406906dba511c22fd813a4a90dab660a3ef33aac792239709f12252d3c8ad52d8d7a84d2d5925660524 + checksum: c1295321b7767c726428db155d417cdc3459532746bf11b18c5ea296c128137379570a7f83cd1da012b8d64fb0228b9e75f9b66fdd7249f945f020c901bcdb9d languageName: node linkType: hard @@ -1958,10 +1959,10 @@ __metadata: linkType: hard "@aws-sdk/s3-presigned-post@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/s3-presigned-post@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/s3-presigned-post@npm:3.670.0" dependencies: - "@aws-sdk/client-s3": 3.668.0 + "@aws-sdk/client-s3": 3.670.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-format-url": 3.667.0 "@smithy/middleware-endpoint": ^3.1.4 @@ -1970,15 +1971,15 @@ __metadata: "@smithy/util-hex-encoding": ^3.0.0 "@smithy/util-utf8": ^3.0.0 tslib: ^2.6.2 - checksum: 0f739b96258ac3c54de379396449bf2e04f96f9d091998c8b44941566d54d35a75898128fc7c22636b13ce14051ffe2027d93dc3dfc365f5f8fa85548e295e85 + checksum: acdf25a8806e988717ebdfb9bafe8892a3df4c9beee36ed2c6fb6e4c6fc02eac3df43ccb1ae5cfcc1b0ce9f4a74f126efddccd36fc9c445cdf307992b0651abb languageName: node linkType: hard "@aws-sdk/s3-request-presigner@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/s3-request-presigner@npm:3.668.0" + version: 3.670.0 + resolution: "@aws-sdk/s3-request-presigner@npm:3.670.0" dependencies: - "@aws-sdk/signature-v4-multi-region": 3.667.0 + "@aws-sdk/signature-v4-multi-region": 3.669.0 "@aws-sdk/types": 3.667.0 "@aws-sdk/util-format-url": 3.667.0 "@smithy/middleware-endpoint": ^3.1.4 @@ -1986,7 +1987,7 @@ __metadata: "@smithy/smithy-client": ^3.4.0 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: 0349c23c76866ccbc21a5b3253b6a342f8389b4018f460a03dd665ecd6570e8ca1de8728b0888c07361ae8d9bddeb21f6e116928f1d299ac6090181c68635f55 + checksum: 25916ce05fa3c861fd8d37ad3102867d028984688f00e5ea25164c5a2de9f1377b6a268746beae9b1c827b0c607ea9d1222f6afd50111bf064801b9c34d23c13 languageName: node linkType: hard @@ -2006,17 +2007,17 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/signature-v4-multi-region@npm:3.667.0": - version: 3.667.0 - resolution: "@aws-sdk/signature-v4-multi-region@npm:3.667.0" +"@aws-sdk/signature-v4-multi-region@npm:3.669.0": + version: 3.669.0 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.669.0" dependencies: - "@aws-sdk/middleware-sdk-s3": 3.667.0 + "@aws-sdk/middleware-sdk-s3": 3.669.0 "@aws-sdk/types": 3.667.0 "@smithy/protocol-http": ^4.1.4 "@smithy/signature-v4": ^4.2.0 "@smithy/types": ^3.5.0 tslib: ^2.6.2 - checksum: b9eb0041f1d3535170eeb140ebfae21e8bfbc2fa0be0b273eea454c60dfff2f325f38c556054c0ca039c601c3a4765470a29ba4969955d0c5dce5ea838118a12 + checksum: b8f6e28ffb9d24941f574999571ac05b42ce7e28f354ba45837366b653f3bc2af4814d3f1f34688003bfeda0b4cb0483d74c04f5be0b5782f2d82eb740a088a5 languageName: node linkType: hard @@ -2164,14 +2165,14 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-dynamodb@npm:3.668.0, @aws-sdk/util-dynamodb@npm:^3.654.0": - version: 3.668.0 - resolution: "@aws-sdk/util-dynamodb@npm:3.668.0" +"@aws-sdk/util-dynamodb@npm:3.670.0, @aws-sdk/util-dynamodb@npm:^3.654.0": + version: 3.670.0 + resolution: "@aws-sdk/util-dynamodb@npm:3.670.0" dependencies: tslib: ^2.6.2 peerDependencies: - "@aws-sdk/client-dynamodb": ^3.668.0 - checksum: f592e84e78abd484111bdfa67e0e0886a04f1245c0292090865d9d61042a0094c36c9dd9133486bb2c4a1fb00a837b4d8ab0ddc70b0ef36c227b11b5b9d54e68 + "@aws-sdk/client-dynamodb": ^3.670.0 + checksum: 8a9f9fa1913cb7d526dba7de592669acd6249abe0d814e961e89374d0a14e8ba75bd52088dbe4ed8ac523ea557d73b6062c681bffa3d3d4d473f2eee6e27429b languageName: node linkType: hard @@ -2237,15 +2238,15 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-user-agent-browser@npm:3.667.0": - version: 3.667.0 - resolution: "@aws-sdk/util-user-agent-browser@npm:3.667.0" +"@aws-sdk/util-user-agent-browser@npm:3.670.0": + version: 3.670.0 + resolution: "@aws-sdk/util-user-agent-browser@npm:3.670.0" dependencies: "@aws-sdk/types": 3.667.0 "@smithy/types": ^3.5.0 bowser: ^2.11.0 tslib: ^2.6.2 - checksum: 304094b309461b7e63f143c29e40a7da04f1f49d1ee1fba21fbea801657e5058b4ec2c756ade45dc2e55a373408b079a4426a7c42d605ac77561ca3c22f36bb5 + checksum: 70c4b8374bbd52d573b2e691998e96611dbcad3692b296a0674871ddc4a3ae42bcba1a6130fde3143ec9bcfc537449c197568b95a160996658a73da1ceda7ee6 languageName: node linkType: hard @@ -2260,11 +2261,11 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-user-agent-node@npm:3.668.0": - version: 3.668.0 - resolution: "@aws-sdk/util-user-agent-node@npm:3.668.0" +"@aws-sdk/util-user-agent-node@npm:3.669.0": + version: 3.669.0 + resolution: "@aws-sdk/util-user-agent-node@npm:3.669.0" dependencies: - "@aws-sdk/middleware-user-agent": 3.668.0 + "@aws-sdk/middleware-user-agent": 3.669.0 "@aws-sdk/types": 3.667.0 "@smithy/node-config-provider": ^3.1.8 "@smithy/types": ^3.5.0 @@ -2274,7 +2275,7 @@ __metadata: peerDependenciesMeta: aws-crt: optional: true - checksum: 159a8062e655f8846964a5ee75abc8aa6aff1b96941aee8f6690ade20d7356b75c2aa9aef07d172a6382432d57af4d8e83fe0d9e3a4c99256f213bf5f4087573 + checksum: 89af06914764efbccc2bdd07f8cc26c4949a20dfed30064d71f1e310cfa9b3bf40a94c2245723209a790bdacf71ced9d5c2742760205879417e220f00808add2 languageName: node linkType: hard @@ -2396,15 +2397,15 @@ __metadata: linkType: hard "@babel/generator@npm:^7.25.9, @babel/generator@npm:^7.26.0": - version: 7.26.0 - resolution: "@babel/generator@npm:7.26.0" + version: 7.26.2 + resolution: "@babel/generator@npm:7.26.2" dependencies: - "@babel/parser": ^7.26.0 + "@babel/parser": ^7.26.2 "@babel/types": ^7.26.0 "@jridgewell/gen-mapping": ^0.3.5 "@jridgewell/trace-mapping": ^0.3.25 jsesc: ^3.0.2 - checksum: 3b1edb8202f39e1600eb1342a04571b8ba66148b7165ec3cf7a072696fa81301f373648e19492289aa832e60a42f3ed367ae4b1ae6ad92968393f11a35dae70c + checksum: 6ff850b7d6082619f8c2f518d993cf7254cfbaa20b026282cbef5c9b2197686d076a432b18e36c4d1a42721c016df4f77a8f62c67600775d9683621d534b91b4 languageName: node linkType: hard @@ -2550,9 +2551,9 @@ __metadata: languageName: node linkType: hard -"@babel/helper-define-polyfill-provider@npm:^0.6.1, @babel/helper-define-polyfill-provider@npm:^0.6.2": - version: 0.6.2 - resolution: "@babel/helper-define-polyfill-provider@npm:0.6.2" +"@babel/helper-define-polyfill-provider@npm:^0.6.1, @babel/helper-define-polyfill-provider@npm:^0.6.2, @babel/helper-define-polyfill-provider@npm:^0.6.3": + version: 0.6.3 + resolution: "@babel/helper-define-polyfill-provider@npm:0.6.3" dependencies: "@babel/helper-compilation-targets": ^7.22.6 "@babel/helper-plugin-utils": ^7.22.5 @@ -2561,7 +2562,7 @@ __metadata: resolve: ^1.14.2 peerDependencies: "@babel/core": ^7.4.0 || ^8.0.0-0 <8.0.0 - checksum: 2bba965ea9a4887ddf9c11d51d740ab473bd7597b787d042c325f6a45912dfe908c2d6bb1d837bf82f7e9fa51e6ad5150563c58131d2bb85515e63d971414a9c + checksum: 710e6d8a5391736b9f53f09d0494575c2e03de199ad8d1349bc8e514cb85251ea1f1842c2ff44830849d482052ddb42ae931101002a87a263b12f649c2e57c01 languageName: node linkType: hard @@ -5389,49 +5390,49 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/merge@npm:^9.0.4, @graphql-tools/merge@npm:^9.0.6": - version: 9.0.7 - resolution: "@graphql-tools/merge@npm:9.0.7" +"@graphql-tools/merge@npm:^9.0.4, @graphql-tools/merge@npm:^9.0.8": + version: 9.0.8 + resolution: "@graphql-tools/merge@npm:9.0.8" dependencies: - "@graphql-tools/utils": ^10.5.4 + "@graphql-tools/utils": ^10.5.5 tslib: ^2.4.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: d61a0cead730fd50dcc0055e94d7c53a1a6e982d0fb35d8a5c0721191eec6ab1102fcea2aabbdab0a224bdfd779458e4292b066572b562419b2958b255e41fa7 + checksum: f1441595332068d7fe6c344da5ddd0bbc114fc64858f71a6ba6c766d658e68ef20f03cf387aaba7d5f9cf7e97cbf7ebd30f21c99438b742cdd0ecaf0ccbd5719 languageName: node linkType: hard "@graphql-tools/resolvers-composition@npm:^7.0.1": - version: 7.0.1 - resolution: "@graphql-tools/resolvers-composition@npm:7.0.1" + version: 7.0.2 + resolution: "@graphql-tools/resolvers-composition@npm:7.0.2" dependencies: - "@graphql-tools/utils": ^10.0.13 + "@graphql-tools/utils": ^10.5.5 lodash: 4.17.21 - micromatch: ^4.0.4 + micromatch: ^4.0.8 tslib: ^2.4.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: a1ba75b6128880ff9571945c408c88f26587fd2bd83c6149f5fb98270b0a209f7d6f4aa10d04a394d1037badfd42234fdf1fc9c756b5cb94e2099ee1c5e1e8e5 + checksum: 29223439ea6845b923c75f339568b7f74b137be01bf52ecabb9d79e9f705961e3cea60a770edd598e3c36da61e5f1fdef1ab836bffceda6afd624c81dc885d32 languageName: node linkType: hard "@graphql-tools/schema@npm:^10.0.6": - version: 10.0.6 - resolution: "@graphql-tools/schema@npm:10.0.6" + version: 10.0.7 + resolution: "@graphql-tools/schema@npm:10.0.7" dependencies: - "@graphql-tools/merge": ^9.0.6 - "@graphql-tools/utils": ^10.5.4 + "@graphql-tools/merge": ^9.0.8 + "@graphql-tools/utils": ^10.5.5 tslib: ^2.4.0 value-or-promise: ^1.0.12 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: 3548c7daf7df7a13ae8852379b5589ee2041caabca31e3c14106dfae3e4417b66623a9f33037c93659e84c1129e9ab93ba16138f1fdd43c6c858802d4c9e93a8 + checksum: d0cf9d286755d4d1ed7d7f33b51f1f1fcf9afdcbc3470078f5face1e49253948963b5a5e8cbdcf08fecc7b016aecbac794428c2269c6c9de91422e5b4375cec1 languageName: node linkType: hard -"@graphql-tools/utils@npm:^10.0.13, @graphql-tools/utils@npm:^10.3.1, @graphql-tools/utils@npm:^10.5.4": - version: 10.5.4 - resolution: "@graphql-tools/utils@npm:10.5.4" +"@graphql-tools/utils@npm:^10.3.1, @graphql-tools/utils@npm:^10.5.5": + version: 10.5.5 + resolution: "@graphql-tools/utils@npm:10.5.5" dependencies: "@graphql-typed-document-node/core": ^3.1.1 cross-inspect: 1.0.1 @@ -5439,7 +5440,7 @@ __metadata: tslib: ^2.4.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: 56b41b276401c9010e47627b0d84336ac840d4e3e7c0124884626f11b92a20a1f1aec97712dd06d8adf5239fc39f4a86d4f4349f6a7028205e577e4c200bf070 + checksum: bbc5ea58c286e1a9f82876570f98b29079e224a8b3568ca8df4720de8c790e1cb8772182d1411af99d6ac749b9082810e02f21f0813dc43c4d0438ef9e04e885 languageName: node linkType: hard @@ -5509,12 +5510,12 @@ __metadata: linkType: hard "@iconify/json@npm:^2.2.142": - version: 2.2.260 - resolution: "@iconify/json@npm:2.2.260" + version: 2.2.271 + resolution: "@iconify/json@npm:2.2.271" dependencies: "@iconify/types": "*" pathe: ^1.1.2 - checksum: b2e7c2f0986ac53854ff4144520edd92da1955edb017fc0e5dfb52f4dfd268befbc9a9ca87398830e67ae1e7da6e557c9a79b2598f06498774d55961954f1e27 + checksum: 357f01c02861442ac61d87f4947bb941cf90fdc076ee3b4f14f2cf9ff9f1381b58f461866000490ebf3916a1b61b69bcd0ec0fbe0815cec874b522910377242d languageName: node linkType: hard @@ -12254,14 +12255,17 @@ __metadata: resolution: "@webiny/api-elasticsearch-tasks@workspace:packages/api-elasticsearch-tasks" dependencies: "@webiny/api": 0.0.0 + "@webiny/api-dynamodb-to-elasticsearch": 0.0.0 "@webiny/api-elasticsearch": 0.0.0 "@webiny/api-headless-cms": 0.0.0 "@webiny/api-i18n": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/api-security": 0.0.0 "@webiny/api-tenancy": 0.0.0 "@webiny/api-wcp": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 + "@webiny/db": 0.0.0 "@webiny/db-dynamodb": 0.0.0 "@webiny/error": 0.0.0 "@webiny/handler": 0.0.0 @@ -12699,6 +12703,7 @@ __metadata: "@webiny/api-wcp": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 + "@webiny/db-dynamodb": 0.0.0 "@webiny/error": 0.0.0 "@webiny/handler": 0.0.0 "@webiny/handler-aws": 0.0.0 @@ -15261,6 +15266,7 @@ __metadata: version: 0.0.0-use.local resolution: "@webiny/db@workspace:packages/db" dependencies: + "@webiny/api": 0.0.0 "@webiny/cli": 0.0.0 "@webiny/project-utils": 0.0.0 rimraf: ^5.0.5 @@ -18342,7 +18348,7 @@ __metadata: languageName: node linkType: hard -"browserslist@npm:^4.23.3, browserslist@npm:^4.24.0": +"browserslist@npm:^4.24.0, browserslist@npm:^4.24.2": version: 4.24.2 resolution: "browserslist@npm:4.24.2" dependencies: @@ -18771,9 +18777,9 @@ __metadata: linkType: hard "caniuse-lite@npm:^1.0.30001669": - version: 1.0.30001673 - resolution: "caniuse-lite@npm:1.0.30001673" - checksum: c4e1dce7375d228fa896bb8d94921b817ef0a09df562e78609830216d5f254f247e8ca0f5f42c4078f360f4f5bd4c48d6303e2d5bb7f74561ba30bc14d08fe0a + version: 1.0.30001680 + resolution: "caniuse-lite@npm:1.0.30001680" + checksum: 2641d2b18c5ab0a6663cb350c5adc81e5ede1a7677d1c7518a8053ada87bf6f206419e1820a2608f76fa5e4f7bea327cbe47df423783e571569a88c0ea645270 languageName: node linkType: hard @@ -19888,11 +19894,11 @@ __metadata: linkType: soft "core-js-compat@npm:^3.38.0, core-js-compat@npm:^3.38.1": - version: 3.38.1 - resolution: "core-js-compat@npm:3.38.1" + version: 3.39.0 + resolution: "core-js-compat@npm:3.39.0" dependencies: - browserslist: ^4.23.3 - checksum: a0a5673bcd59f588f0cd0b59cdacd4712b82909738a87406d334dd412eb3d273ae72b275bdd8e8fef63fca9ef12b42ed651be139c7c44c8a1acb423c8906992e + browserslist: ^4.24.2 + checksum: 2d7d087c3271d711d03a55203d4756f6288317a1ce35cdc8bafaf1833ef21fd67a92a50cff8dcf7df1325ac63720906ab3cf514c85b238c95f65fca1040f6ad6 languageName: node linkType: hard @@ -21819,9 +21825,9 @@ __metadata: linkType: hard "electron-to-chromium@npm:^1.5.41": - version: 1.5.47 - resolution: "electron-to-chromium@npm:1.5.47" - checksum: a6f31fdc79d1845365a54dfeaa5684cec6961c8bdeb3c406b520a0bb5987a0eaf28e24a75086ff1093fb629ca1bcfbfbd24d04d0ef69c37d85fedd6c621fef76 + version: 1.5.57 + resolution: "electron-to-chromium@npm:1.5.57" + checksum: fffd6dc9aeca39f94ef083a5a68c7c330994a4a5c31960758af81d743db6ea79d142d0b97fc852700955726c3cb6296de1183051a94118c0978c072e86ac74b7 languageName: node linkType: hard @@ -21841,9 +21847,24 @@ __metadata: languageName: node linkType: hard -"elliptic@npm:^6.5.3, elliptic@npm:^6.5.4, elliptic@npm:^6.5.5": - version: 6.5.7 - resolution: "elliptic@npm:6.5.7" +"elliptic@npm:^6.5.3, elliptic@npm:^6.5.4": + version: 6.5.4 + resolution: "elliptic@npm:6.5.4" + dependencies: + bn.js: ^4.11.9 + brorand: ^1.1.0 + hash.js: ^1.0.0 + hmac-drbg: ^1.0.1 + inherits: ^2.0.4 + minimalistic-assert: ^1.0.1 + minimalistic-crypto-utils: ^1.0.1 + checksum: d56d21fd04e97869f7ffcc92e18903b9f67f2d4637a23c860492fbbff5a3155fd9ca0184ce0c865dd6eb2487d234ce9551335c021c376cd2d3b7cb749c7d10f4 + languageName: node + linkType: hard + +"elliptic@npm:^6.5.5": + version: 6.5.5 + resolution: "elliptic@npm:6.5.5" dependencies: bn.js: ^4.11.9 brorand: ^1.1.0 @@ -21852,7 +21873,7 @@ __metadata: inherits: ^2.0.4 minimalistic-assert: ^1.0.1 minimalistic-crypto-utils: ^1.0.1 - checksum: af0ffddffdbc2fea4eeec74388cd73e62ed5a0eac6711568fb28071566319785df529c968b0bf1250ba4bc628e074b2d64c54a633e034aa6f0c6b152ceb49ab8 + checksum: ec9105e4469eb3b32b0ee2579756c888ddf3f99d259aa0d65fccb906ee877768aaf8880caae73e3e669c9a4adeb3eb1945703aa974ec5000d2d33a239f4567eb languageName: node linkType: hard @@ -23235,7 +23256,7 @@ __metadata: fast-content-type-parse: ^1.0.0 find-my-way: ^7.6.0 light-my-request: ^5.6.1 - pino: ^8.21.0 + pino: ^8.5.0 process-warning: ^2.0.0 proxy-addr: ^2.0.7 rfdc: ^1.3.0 @@ -25116,8 +25137,8 @@ __metadata: linkType: hard "http-proxy-middleware@npm:^2.0.3": - version: 2.0.7 - resolution: "http-proxy-middleware@npm:2.0.7" + version: 2.0.6 + resolution: "http-proxy-middleware@npm:2.0.6" dependencies: "@types/http-proxy": ^1.17.8 http-proxy: ^1.18.1 @@ -25129,7 +25150,7 @@ __metadata: peerDependenciesMeta: "@types/express": optional: true - checksum: 18caa21145917aa1054740353916e8f03f5a3a93bede9106f1f44d84f7b174df17af1c72bf5fade5cc440c2058ee813f47cbb2bdd6ae6874af1cf33e0ac575f3 + checksum: 2ee85bc878afa6cbf34491e972ece0f5be0a3e5c98a60850cf40d2a9a5356e1fc57aab6cff33c1fc37691b0121c3a42602d2b1956c52577e87a5b77b62ae1c3a languageName: node linkType: hard @@ -28929,6 +28950,16 @@ __metadata: languageName: node linkType: hard +"micromatch@npm:^4.0.8": + version: 4.0.8 + resolution: "micromatch@npm:4.0.8" + dependencies: + braces: ^3.0.3 + picomatch: ^2.3.1 + checksum: 79920eb634e6f400b464a954fcfa589c4e7c7143209488e44baf627f9affc8b1e306f41f4f0deedde97e69cb725920879462d3e750ab3bd3c1aed675bb3a8966 + languageName: node + linkType: hard + "miller-rabin@npm:^4.0.0": version: 4.0.1 resolution: "miller-rabin@npm:4.0.1" @@ -30297,9 +30328,9 @@ __metadata: linkType: hard "object-inspect@npm:^1.13.1": - version: 1.13.2 - resolution: "object-inspect@npm:1.13.2" - checksum: 9f850b3c045db60e0e97746e809ee4090d6ce62195af17dd1e9438ac761394a7d8ec4f7906559aea5424eaf61e35d3e53feded2ccd5f62fcc7d9670d3c8eb353 + version: 1.13.3 + resolution: "object-inspect@npm:1.13.3" + checksum: 8c962102117241e18ea403b84d2521f78291b774b03a29ee80a9863621d88265ffd11d0d7e435c4c2cea0dc2a2fbf8bbc92255737a05536590f2df2e8756f297 languageName: node linkType: hard @@ -31360,7 +31391,7 @@ __metadata: languageName: node linkType: hard -"pino-abstract-transport@npm:^1.0.0": +"pino-abstract-transport@npm:^1.0.0, pino-abstract-transport@npm:v1.0.0": version: 1.0.0 resolution: "pino-abstract-transport@npm:1.0.0" dependencies: @@ -31470,6 +31501,27 @@ __metadata: languageName: node linkType: hard +"pino@npm:^8.5.0": + version: 8.9.0 + resolution: "pino@npm:8.9.0" + dependencies: + atomic-sleep: ^1.0.0 + fast-redact: ^3.1.1 + on-exit-leak-free: ^2.1.0 + pino-abstract-transport: v1.0.0 + pino-std-serializers: ^6.0.0 + process-warning: ^2.0.0 + quick-format-unescaped: ^4.0.3 + real-require: ^0.2.0 + safe-stable-stringify: ^2.3.1 + sonic-boom: ^3.1.0 + thread-stream: ^2.0.0 + bin: + pino: bin.js + checksum: c7eb9345280383b9af4661b34ae2156c94e535eb33c8deaf525e00212fa9e4f0b9797cdeda17b80b333325d8a0aec6e7566618071506e944736822022b671fa8 + languageName: node + linkType: hard + "pirates@npm:^4.0.1, pirates@npm:^4.0.4": version: 4.0.5 resolution: "pirates@npm:4.0.5" @@ -34696,7 +34748,7 @@ __metadata: "@babel/code-frame": ^7.26.2 "@babel/compat-data": ^7.26.2 "@babel/core": ^7.26.0 - "@babel/helper-define-polyfill-provider": ^0.6.2 + "@babel/helper-define-polyfill-provider": ^0.6.3 "@babel/helper-environment-visitor": ^7.24.7 "@babel/parser": ^7.26.2 "@babel/plugin-proposal-class-properties": ^7.18.6 @@ -35665,7 +35717,7 @@ __metadata: languageName: node linkType: hard -"sonic-boom@npm:^3.0.0": +"sonic-boom@npm:^3.0.0, sonic-boom@npm:^3.1.0": version: 3.2.1 resolution: "sonic-boom@npm:3.2.1" dependencies: @@ -36892,6 +36944,15 @@ __metadata: languageName: node linkType: hard +"thread-stream@npm:^2.0.0": + version: 2.3.0 + resolution: "thread-stream@npm:2.3.0" + dependencies: + real-require: ^0.2.0 + checksum: e9ea58f9f36320165b41c2aae5c439bf68bd3575eb533c458483d8b290e31d519979e351408c7d6e248711611434332c2a3aae2165650b028cc3eb9b1052ac16 + languageName: node + linkType: hard + "thread-stream@npm:^2.6.0": version: 2.7.0 resolution: "thread-stream@npm:2.7.0" @@ -36967,9 +37028,9 @@ __metadata: linkType: hard "tiny-lru@npm:^10.0.0": - version: 10.4.1 - resolution: "tiny-lru@npm:10.4.1" - checksum: ec2ce11278ff5ecd094823d9d6fcc16fe0aa250bf25cb533d40b997452a113a9ecd9cccbf503ac459a17bb812d94c5ef05cbdfe94b5b8ad03d7cf4fadb78a905 + version: 10.0.1 + resolution: "tiny-lru@npm:10.0.1" + checksum: 58b5f17a357625335aa3b90ee8c9b3e9abede5c1f46066c73deb129574a205efb112807d6d473909e73f1d874ea99bf14eb5c88223d540eb32ebb5e1ff146689 languageName: node linkType: hard