Replace nedb with fork @seald-io/nedb (#1425)

* Replace nedb with fork `@seald-io/nedb`

* fix integration tests and load async

* add rush change file

* fixes and update pnpm-lock.yaml after rebase

* update pnpm-lock.yaml

* fix integration test

* Update pnmp-lock.yaml after rebasing

---------

Co-authored-by: Castro, Mario <[email protected]>

common/changes/@boostercloud/framework-core/upgrade_nedb_seald_2023-07-28-14-42.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@boostercloud/framework-core",
+      "comment": "replace nedb with seald",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@boostercloud/framework-core"
+}

packages/framework-integration-tests/integration/provider-specific/local/commands.integration.ts

@@ -2,19 +2,19 @@ import { ApolloClient, NormalizedCacheObject } from '@apollo/client'
 import { changeCartItem, graphQLClient } from './utils'
 import { random } from 'faker'
 import { expect } from 'chai'
-import * as DataStore from 'nedb'
+import Datastore from '@seald-io/nedb'
 import { sandboxPath } from './constants'
 import * as util from 'util'
 import * as path from 'path'
 import { waitForIt } from '../../helper/sleep'
 
 describe('commands', () => {
-  let events: DataStore<unknown>
+  let events: Datastore<unknown>
 
   let client: ApolloClient<NormalizedCacheObject>
 
   before(async () => {
-    events = new DataStore(path.join(sandboxPath, '.booster', 'events.json'))
+    events = new Datastore(path.join(sandboxPath, '.booster', 'events.json'))
     client = await graphQLClient()
   })
 

packages/framework-integration-tests/integration/provider-specific/local/read-models.integration.ts

@@ -2,18 +2,18 @@ import { ApolloClient, NormalizedCacheObject } from '@apollo/client'
 import { changeCartItem, graphQLClient } from './utils'
 import { random } from 'faker'
 import { expect } from 'chai'
-import * as DataStore from 'nedb'
+import Datastore from '@seald-io/nedb'
 import { sandboxPath } from './constants'
 import * as path from 'path'
 import { waitForIt } from '../../helper/sleep'
 
 describe('read-models', () => {
-  let readModels: DataStore<unknown>
+  let readModels: Datastore<unknown>
 
   let client: ApolloClient<NormalizedCacheObject>
 
   before(async () => {
-    readModels = new DataStore(path.join(sandboxPath, '.booster', 'read_models.json'))
+    readModels = new Datastore(path.join(sandboxPath, '.booster', 'read_models.json'))
     client = await graphQLClient()
   })
 

packages/framework-integration-tests/integration/provider-specific/local/start/setup.ts

@@ -27,6 +27,6 @@ before(async () => {
     throw new Error('Pid not found')
   }
   storePIDFor(sandboxPath, serverProcess.pid) //store pid to kill process on stop
-  await sleep(2000)
+  await sleep(10000)
   console.log('local server ready')
 })

packages/framework-integration-tests/integration/provider-unaware/end-to-end/commands.integration.ts

@@ -1,6 +1,6 @@
 import { ApolloClient, NormalizedCacheObject, gql } from '@apollo/client'
 import { commerce, finance, random } from 'faker'
-import { expect } from 'chai'
+import { expect } from '../../helper/expect'
 import { applicationUnderTest } from './setup'
 
 describe('Commands end-to-end tests', () => {

packages/framework-integration-tests/integration/provider-unaware/end-to-end/entities.integration.ts

@@ -1,14 +1,12 @@
 import { ApolloClient, NormalizedCacheObject, gql } from '@apollo/client'
 import { random, commerce, finance, lorem, internet } from 'faker'
 import { expect } from 'chai'
-import { sleep, waitForIt } from '../../helper/sleep'
+import { waitForIt } from '../../helper/sleep'
 import { applicationUnderTest } from './setup'
 import { UUID } from '@boostercloud/framework-types'
 import { NEW_CART_IDS, QUANTITY_AFTER_DATA_MIGRATION_V2, QUANTITY_TO_MIGRATE_DATA } from '../../../src/constants'
 import { ProductType } from '../../../src/entities/product'
 
-const secs = 10
-
 describe('Entities end-to-end tests', () => {
   let client: ApolloClient<NormalizedCacheObject>
   let userToken: string
@@ -147,9 +145,6 @@ describe('Entities end-to-end tests', () => {
         `,
       })
 
-      console.log(`Waiting ${secs} second${secs > 1 ? 's' : ''} for deletion to complete...`)
-      await sleep(secs * 1000)
-
       client = applicationUnderTest.graphql.client(userToken)
       // Retrieve updated entity
       const queryResult = await waitForIt(
@@ -178,7 +173,7 @@ describe('Entities end-to-end tests', () => {
             `,
           })
         },
-        () => true
+        (result) => !result?.data?.ProductReadModel
       )
 
       const productData = queryResult.data.ProductReadModel
@@ -331,10 +326,9 @@ describe('Entities end-to-end tests', () => {
             })
           },
           (result) => {
-            const resultReadModels = result?.data?.ListDataMigrationsReadModels
-            const count = resultReadModels?.count
+            const count = result?.data?.ListDataMigrationsReadModels?.count
             if (count < 2) {
-              return `Waiting for at least 2 migrations. Done ${count} migrations. ${JSON.stringify(resultReadModels)}`
+              return `Waiting for ${count} migrations. Done ${count} migrations`
             }
             return true
           }

packages/framework-integration-tests/integration/provider-unaware/end-to-end/read-models.integration.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/ban-ts-comment */
 import { ApolloClient, NormalizedCacheObject, gql } from '@apollo/client'
 import { commerce, finance, internet, lorem, random } from 'faker'
-import { expect } from 'chai'
+import { expect } from '../../helper/expect'
 import { waitForIt } from '../../helper/sleep'
 import { CartItem } from '../../../src/common/cart-item'
 import { applicationUnderTest } from './setup'
@@ -179,12 +179,7 @@ describe('Read models end-to-end tests', () => {
         await Promise.all(changeCartPromises)
       })
 
-      // TODO this test is failing in local because of local provider doesn't provides optimistic concurrency control
-      // TODO Remove condition when it will be fixed
       it('should retrieve expected cart', async () => {
-        if (process.env.TESTED_PROVIDER === 'LOCAL') {
-          return
-        }
         const queryResult = await waitForIt(
           () => {
             return client.query({

packages/framework-integration-tests/integration/provider-unaware/end-to-end/subscriptions.integration.ts → packages/framework-integration-tests/integration/provider-unaware/functionality/subscriptions.integration.ts

@@ -257,7 +257,7 @@ describe('subscriptions', () => {
         })
       })
     })
-    
+
     context('with a user without the required role', () => {
       let loggedClient: DisconnectableApolloClient
 

packages/framework-integration-tests/package.json

@@ -20,6 +20,7 @@
     "tslib": "^2.4.0",
     "@effect-ts/core": "^0.60.4",
     "express-unless": "2.1.3",
+    "express": "^4.17.1",
     "effect": "^2.4.6",
     "@effect/platform": "^0.48.3",
     "@effect/schema": "^0.64.2",
@@ -73,7 +74,7 @@
     "jwks-rsa": "3.0.1",
     "mocha": "10.2.0",
     "mocha-skip-if": "0.0.3",
-    "nedb": "^1.8.0",
+    "@seald-io/nedb": "4.0.2",
     "nyc": "^15.1.0",
     "prettier": "2.3.0",
     "rimraf": "^5.0.0",
@@ -110,7 +111,7 @@
     "integration/aws-end-to-end": "echo 'Skipping integration tests.' # TODO 'Enable it again when AWS Provider will be upgrade it' TS_NODE_PROJECT=\"./tsconfig.integration.json\" TESTED_PROVIDER=AWS AWS_SDK_LOAD_CONFIG=true BOOSTER_ENV=production mocha --forbid-only --exit --config \"integration/provider-unaware/end-to-end/.mocharc.yml\" \"integration/provider-unaware/end-to-end/**/*.integration.ts\"",
     "integration/aws-load": "echo 'Skipping integration tests.' # TS_NODE_PROJECT=\"./tsconfig.integration.json\" TESTED_PROVIDER=AWS AWS_SDK_LOAD_CONFIG=true BOOSTER_ENV=production mocha --forbid-only --exit --config \"integration/provider-unaware/load/.mocharc.yml\" \"integration/provider-unaware/load/**/*.load.ts\"",
     "integration/aws-nuke": "TS_NODE_PROJECT=\"./tsconfig.integration.json\" AWS_SDK_LOAD_CONFIG=true BOOSTER_ENV=production mocha --forbid-only --exit --config \"integration/provider-specific/aws/nuke/.mocharc.yml\" \"integration/provider-specific/aws/nuke/**/*.integration.ts\"",
-    "integration/local": "TS_NODE_PROJECT=\"./tsconfig.integration.json\" BOOSTER_ENV=local mocha --forbid-only --exit --config \"integration/provider-specific/local/.mocharc.yml\" \"integration/provider-specific/local/**/*.integration.ts\"",
+    "integration/local": "TS_NODE_PROJECT=\"./tsconfig.integration.json\" npm run integration/local-start && npm run integration/local-end-to-end && npm run integration/local-stop",
     "integration/local-ongoing": "TS_NODE_PROJECT=\"./tsconfig.integration.json\" npm run integration/local-start && npm run integration/local-stop",
     "integration/local-start": "TS_NODE_PROJECT=\"./tsconfig.integration.json\" BOOSTER_ENV=local mocha --forbid-only --exit --config \"integration/provider-specific/local/start/.mocharc.yml\" \"integration/provider-specific/local/start/*.integration.ts\"",
     "integration/local-func": "TS_NODE_PROJECT=\"./tsconfig.integration.json\" TESTED_PROVIDER=LOCAL BOOSTER_ENV=local mocha --forbid-only --exit --config \"integration/provider-unaware/functionality/.mocharc.yml\" \"integration/provider-unaware/functionality/**/*.integration.ts\"",

packages/framework-provider-local/package.json

@@ -25,8 +25,7 @@
   "dependencies": {
     "@boostercloud/framework-common-helpers": "workspace:^2.8.0",
     "@boostercloud/framework-types": "workspace:^2.8.0",
-    "@types/nedb": "^1.8.12",
-    "nedb": "^1.8.0",
+    "@seald-io/nedb": "4.0.2",
     "tslib": "^2.4.0",
     "@effect-ts/core": "^0.60.4",
     "ws": "8.12.0"

packages/framework-provider-local/src/library/health-adapter.ts

@@ -1,22 +1,17 @@
-import * as DataStore from 'nedb'
 import { EventRegistry, ReadModelRegistry } from '../services'
 import { eventsDatabase, readModelsDatabase } from '../paths'
 import { boosterLocalPort, HealthEnvelope, UUID } from '@boostercloud/framework-types'
 import { existsSync } from 'fs'
 import * as express from 'express'
 import { request } from '@boostercloud/framework-common-helpers'
+import Nedb from '@seald-io/nedb'
 
 export async function databaseUrl(): Promise<Array<string>> {
   return [eventsDatabase, readModelsDatabase]
 }
 
-export async function countAll(database: DataStore): Promise<number> {
-  const count = await new Promise<number>((resolve, reject) => {
-    database.count({}, (err, docs) => {
-      if (err) reject(err)
-      else resolve(docs)
-    })
-  })
+export async function countAll(database: Nedb): Promise<number> {
+  const count = await database.countAsync({})
   return count ?? 0
 }
 

packages/framework-provider-local/src/services/event-registry.ts

@@ -1,16 +1,26 @@
 /* eslint-disable @typescript-eslint/ban-types */
 import { EntitySnapshotEnvelope, EventEnvelope, EventStoreEntryEnvelope } from '@boostercloud/framework-types'
-import * as DataStore from 'nedb'
 import { eventsDatabase } from '../paths'
+const DataStore = require('@seald-io/nedb')
 
 export class EventRegistry {
-  public readonly events: DataStore<EventStoreEntryEnvelope> = new DataStore(eventsDatabase)
+  public readonly events
+  public isLoaded = false
+
   constructor() {
-    this.events.loadDatabase()
+    this.events = new DataStore({ filename: eventsDatabase })
+  }
+
+  async loadDatabaseIfNeeded(): Promise<void> {
+    if (!this.isLoaded) {
+      this.isLoaded = true
+      await this.events.loadDatabaseAsync()
+    }
   }
 
   getCursor(query: object, createdAt = 1, projections?: unknown) {
-    return this.events.find(query, projections).sort({ createdAt: createdAt })
+    const cursor = this.events.findAsync(query, projections)
+    return cursor.sort({ createdAt: createdAt })
   }
 
   public async query(
@@ -19,53 +29,36 @@ export class EventRegistry {
     limit?: number,
     projections?: unknown
   ): Promise<EventStoreEntryEnvelope[]> {
-    const cursor = this.getCursor(query, createdAt, projections)
+    await this.loadDatabaseIfNeeded()
+    let cursor = this.getCursor(query, createdAt, projections)
     if (limit) {
-      cursor.limit(Number(limit))
+      cursor = cursor.limit(Number(limit))
     }
-    const queryPromise = await new Promise<EventStoreEntryEnvelope[]>((resolve, reject) => {
-      cursor.exec((err, docs) => {
-        if (err) reject(err)
-        else resolve(docs)
-      })
-    })
-
-    return queryPromise
+    return await cursor.execAsync()
   }
 
   public async queryLatestSnapshot(query: object): Promise<EntitySnapshotEnvelope | undefined> {
-    const results = await new Promise<EventStoreEntryEnvelope[]>((resolve, reject) =>
-      this.events
-        .find({ ...query, kind: 'snapshot' })
-        .sort({ snapshottedEventCreatedAt: -1 }) // Sort in descending order (newer timestamps first)
-        .exec((err, docs) => {
-          if (err) reject(err)
-          else resolve(docs)
-        })
-    )
-
+    await this.loadDatabaseIfNeeded()
+    const cursor = this.events.findAsync({ ...query, kind: 'snapshot' }).sort({ snapshottedEventCreatedAt: -1 }) // Sort in descending order (newer timestamps first)
+    const results = await cursor.execAsync()
     if (results.length <= 0) {
       return undefined
     }
     return results[0] as EntitySnapshotEnvelope
   }
 
   public async store(storableObject: EventEnvelope | EntitySnapshotEnvelope): Promise<void> {
-    return new Promise((resolve, reject) => {
-      this.events.insert(storableObject, (err) => {
-        err ? reject(err) : resolve()
-      })
-    })
+    await this.loadDatabaseIfNeeded()
+    await this.events.insertAsync(storableObject)
   }
 
   public async deleteAll(): Promise<number> {
-    const deletePromise = new Promise((resolve, reject) =>
-      this.events.remove({}, { multi: true }, (err, numRemoved: number) => {
-        if (err) reject(err)
-        else resolve(numRemoved)
-      })
-    )
+    await this.loadDatabaseIfNeeded()
+    return await this.events.removeAsync({}, { multi: true })
+  }
 
-    return (await deletePromise) as number
+  public async count(query?: object): Promise<number> {
+    await this.loadDatabaseIfNeeded()
+    return await this.events.countAsync(query)
   }
 }