From 675bc6af9308d1f77ec33e966531db11651a1e0d Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Mon, 21 Nov 2022 18:38:34 -0800 Subject: [PATCH 01/15] feat!: [BREAKING] Restructured API for project abstraction (#172) --- api/proto | 2 +- src/__tests__/consumables/cursor.spec.ts | 4 +- src/__tests__/tigris.rpc.spec.ts | 319 ++++------------------- src/__tests__/tigris.schema.spec.ts | 18 +- src/consumables/utils.ts | 24 -- src/db.ts | 35 +-- src/index.ts | 1 - src/tigris.ts | 94 ++----- src/topic.ts | 309 ---------------------- src/types.ts | 17 -- src/utility.ts | 22 +- 11 files changed, 111 insertions(+), 734 deletions(-) delete mode 100644 src/consumables/utils.ts delete mode 100644 src/topic.ts diff --git a/api/proto b/api/proto index 49dd117..8699c98 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit 49dd117755e390fa01dfc48b64fed2f44d02bc67 +Subproject commit 8699c9865c135f40850e7b17c734831758962763 diff --git a/src/__tests__/consumables/cursor.spec.ts b/src/__tests__/consumables/cursor.spec.ts index d6a094a..9335f90 100644 --- a/src/__tests__/consumables/cursor.spec.ts +++ b/src/__tests__/consumables/cursor.spec.ts @@ -30,8 +30,8 @@ describe("class FindCursor", () => { } } ); - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - db = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + db = tigris.getDatabase(); done(); }); diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index 0c0f000..85909cb 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -2,14 +2,12 @@ import {Server, ServerCredentials} from "@grpc/grpc-js"; import {TigrisService} from "../proto/server/v1/api_grpc_pb"; import TestService, {TestTigrisService} from "./test-service"; import { - DatabaseOptions, DeleteRequestOptions, LogicalOperator, SelectorFilterOperator, TigrisCollectionType, TigrisDataTypes, - TigrisSchema, TigrisTopicSchema, - TigrisTopicType, + TigrisSchema, UpdateFieldsOperator, UpdateRequestOptions } from "../types"; @@ -18,7 +16,6 @@ import {Case, Collation, SearchRequest, SearchRequestOptions} from "../search/ty import {Utility} from "../utility"; import {ObservabilityService} from "../proto/server/v1/observability_grpc_pb"; import TestObservabilityService from "./test-observability-service"; -import {Readable} from "node:stream"; import {capture, spy } from "ts-mockito"; describe("rpc tests", () => { @@ -55,56 +52,15 @@ describe("rpc tests", () => { done(); }); - it("listDatabase", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const listDbsPromise = tigris.listDatabases(); - listDbsPromise - .then((value) => { - expect(value.length).toBe(5); - expect(value[0].name).toBe("db1"); - expect(value[1].name).toBe("db2"); - expect(value[2].name).toBe("db3"); - expect(value[3].name).toBe("db4"); - expect(value[4].name).toBe("db5"); - }, - ); - - return listDbsPromise; - }); - - it("createDatabaseIfNotExists", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const dbCreationPromise = tigris.createDatabaseIfNotExists("db6", new DatabaseOptions()); - dbCreationPromise - .then((value) => { - expect(value.db).toBe("db6"); - }, - ); - - return dbCreationPromise; - }); - - it("dropDatabase", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const dbDropPromise = tigris.dropDatabase("db6", new DatabaseOptions()); - dbDropPromise - .then((value) => { - expect(value.status).toBe("dropped"); - expect(value.message).toBe("db6 dropped successfully"); - }, - ); - return dbDropPromise; - }); - it("getDatabase", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db1"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db1"}); + const db1 = tigris.getDatabase(); expect(db1.db).toBe("db1"); }); it("listCollections1", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db1"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db1"}); + const db1 = tigris.getDatabase(); const listCollectionPromise = db1.listCollections(); listCollectionPromise.then(value => { @@ -119,8 +75,8 @@ describe("rpc tests", () => { }); it("listCollections2", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const listCollectionPromise = db1.listCollections(); listCollectionPromise.then(value => { @@ -135,8 +91,8 @@ describe("rpc tests", () => { }); it("describeDatabase", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const databaseDescriptionPromise = db1.describe(); databaseDescriptionPromise.then(value => { @@ -152,8 +108,8 @@ describe("rpc tests", () => { }); it("dropCollection", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const dropCollectionPromise = db1.dropCollection("db3_coll_2"); dropCollectionPromise.then(value => { @@ -164,15 +120,15 @@ describe("rpc tests", () => { }); it("getCollection", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const books = db1.getCollection("books"); expect(books.collectionName).toBe("books"); }); it("insert", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const insertionPromise = db1.getCollection("books").insertOne({ author: "author name", id: 0, @@ -186,8 +142,8 @@ describe("rpc tests", () => { }); it("insert2", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const insertionPromise = db1.getCollection("books").insertOne({ id: 0, title: "science book", @@ -203,8 +159,8 @@ describe("rpc tests", () => { }); it("insertWithOptionalField", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const randomNumber: number = Math.floor(Math.random() * 100); // pass the random number in author field. mock server reads author and sets as the // primaryKey field. @@ -220,8 +176,8 @@ describe("rpc tests", () => { }); it("insertOrReplace", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const insertOrReplacePromise = db1.getCollection("books").insertOrReplaceOne({ author: "author name", id: 0, @@ -235,8 +191,8 @@ describe("rpc tests", () => { }); it("insertOrReplaceWithOptionalField", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const randomNumber: number = Math.floor(Math.random() * 100); // pass the random number in author field. mock server reads author and sets as the // primaryKey field. @@ -252,8 +208,8 @@ describe("rpc tests", () => { }); it("delete", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const deletionPromise = db1.getCollection("books").deleteMany({ op: SelectorFilterOperator.EQ, fields: { @@ -267,8 +223,8 @@ describe("rpc tests", () => { }); it("deleteOne", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const collection = tigris.getDatabase("db3").getCollection("books"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const collection = tigris.getDatabase().getCollection("books"); const spyCollection = spy(collection); const expectedFilter = {id: 1}; @@ -291,8 +247,8 @@ describe("rpc tests", () => { }); it("update", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const updatePromise = db1.getCollection("books").updateMany( { op: SelectorFilterOperator.EQ, @@ -314,8 +270,8 @@ describe("rpc tests", () => { }); it("updateOne", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const collection = tigris.getDatabase("db3").getCollection("books"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const collection = tigris.getDatabase().getCollection("books"); const spyCollection = spy(collection); const expectedFilter = {id: 1}; @@ -341,8 +297,8 @@ describe("rpc tests", () => { }); it("readOne", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const readOnePromise = db1.getCollection("books").findOne( { op: SelectorFilterOperator.EQ, fields: { @@ -360,8 +316,8 @@ describe("rpc tests", () => { }); it("readOneRecordNotFound", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const readOnePromise = db1.getCollection("books").findOne({ op: SelectorFilterOperator.EQ, fields: { @@ -375,8 +331,8 @@ describe("rpc tests", () => { }); it("readOneWithLogicalFilter", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db1 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); const readOnePromise: Promise = db1.getCollection("books").findOne({ op: LogicalOperator.AND, selectorFilters: [ @@ -405,8 +361,8 @@ describe("rpc tests", () => { }); describe("findMany", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db = tigris.getDatabase(); it("with filter using for await on cursor", async () => { const cursor = db.getCollection("books").findMany({ @@ -464,8 +420,8 @@ describe("rpc tests", () => { }); it("search", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db3 = tigris.getDatabase(); const options: SearchRequestOptions = { page: 2, perPage: 12 @@ -490,8 +446,8 @@ describe("rpc tests", () => { }); it("searchStream using iteration", async () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db3 = tigris.getDatabase(); const request: SearchRequest = { q: "philosophy", facets: { @@ -511,8 +467,8 @@ describe("rpc tests", () => { }); it("searchStream using next", async () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db3 = tigris.getDatabase(); const request: SearchRequest = { q: "philosophy", facets: { @@ -534,8 +490,8 @@ describe("rpc tests", () => { }); it("beginTx", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db3 = tigris.getDatabase(); const beginTxPromise = db3.beginTransaction(); beginTxPromise.then(value => { expect(value.id).toBe("id-test"); @@ -545,8 +501,8 @@ describe("rpc tests", () => { }); it("commitTx", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db3 = tigris.getDatabase(); const beginTxPromise = db3.beginTransaction(); beginTxPromise.then(session => { const commitTxResponse = session.commit(); @@ -558,8 +514,8 @@ describe("rpc tests", () => { }); it("rollbackTx", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db3 = tigris.getDatabase(); const beginTxPromise = db3.beginTransaction(); beginTxPromise.then(session => { const rollbackTransactionResponsePromise = session.rollback(); @@ -571,8 +527,8 @@ describe("rpc tests", () => { }); it("transact", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const txDB = tigris.getDatabase("test-tx"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "test-tx"}); + const txDB = tigris.getDatabase(); const books = txDB.getCollection("books"); txDB.transact(tx => { books.insertOne( @@ -617,8 +573,8 @@ describe("rpc tests", () => { }); it("createOrUpdateCollections", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db3 = tigris.getDatabase("db3"); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT,projectName: "db3"}); + const db3 = tigris.getDatabase(); const bookSchema: TigrisSchema = { id: { type: TigrisDataTypes.INT64, @@ -645,31 +601,6 @@ describe("rpc tests", () => { }); }); - it("createOrUpdateTopic", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const alertSchema: TigrisTopicSchema = { - id: { - type: TigrisDataTypes.INT64, - key: { - order: 1 - } - }, - name: { - type: TigrisDataTypes.STRING, - key: { - order: 2 - } - }, - text: { - type: TigrisDataTypes.STRING - } - }; - return db.createOrUpdateTopic("alerts", alertSchema).then(value => { - expect(value).toBeDefined(); - }); - }); - it("serverMetadata", () => { const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); const serverMetadataPromise = tigris.getServerMetadata(); @@ -678,140 +609,6 @@ describe("rpc tests", () => { }); return serverMetadataPromise; }); - - it("publish", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const topic = db.getTopic("test_topic"); - expect(topic.topicName).toBe("test_topic"); - - const promise = topic.publish({ - id: 34, - text: "test" - }); - - promise.then(alert => { - expect(alert.id).toBe(34); - expect(alert.text).toBe("test"); - }); - - return promise; - }); - - it("subscribe using callback", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const topic = db.getTopic("test_topic"); - let success = true; - const expectedIds = new Set(TestTigrisService.ALERTS_B64_BY_ID.keys()); - - topic.subscribe({ - onNext(alert: Alert) { - expect(expectedIds).toContain(alert.id); - expectedIds.delete(alert.id); - }, - onEnd() { - expect(success).toBe(true); - done(); - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - onError(error: Error) { - success = false; - } - }); - }); - - it("subscribe using stream", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const topic = db.getTopic("test_topic"); - const subscription: Readable = topic.subscribe() as Readable; - const expectedIds = new Set(TestTigrisService.ALERTS_B64_BY_ID.keys()); - let success = true; - - subscription.on("data", (alert) =>{ - expect(expectedIds).toContain(alert.id); - expectedIds.delete(alert.id); - }); - subscription.on("error", () => { - success = false; - }); - subscription.on("end", () => { - expect(success).toBe(true); - done(); - }); - }); - - it("subscribeWithFilter", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const topic = db.getTopic("test_topic"); - let success = true; - const expectedIds = new Set(TestTigrisService.ALERTS_B64_BY_ID.keys()); - - topic.subscribeWithFilter({ - op: SelectorFilterOperator.EQ, - fields: { - text: "test" - } - }, - { - onNext(alert: Alert) { - expect(expectedIds).toContain(alert.id); - expectedIds.delete(alert.id); - }, - onEnd() { - expect(success).toBe(true); - done(); - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - onError(error: Error) { - success = false; - } - }); - }); - - it("subscribeToPartitions", (done) => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const topic = db.getTopic("test_topic"); - let success = true; - const expectedIds = new Set(TestTigrisService.ALERTS_B64_BY_ID.keys()); - - const partitions = new Array(); - partitions.push(55); - - topic.subscribeToPartitions(partitions,{ - onNext(alert: Alert) { - expect(expectedIds).toContain(alert.id); - expectedIds.delete(alert.id); - }, - onEnd() { - expect(success).toBe(true); - done(); - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - onError(error: Error) { - success = false; - } - }); - }); - - it("findMany in topic", async () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); - const db = tigris.getDatabase("test_db"); - const topic = db.getTopic("test_topic"); - const expectedIds = new Set(TestTigrisService.ALERTS_B64_BY_ID.keys()); - let seenAlerts = 0; - - for await (const alert of topic.findMany()) { - expect(expectedIds).toContain(alert.id); - expectedIds.delete(alert.id); - seenAlerts++; - } - - expect(seenAlerts).toBe(2); - }); }); export interface IBook extends TigrisCollectionType { @@ -833,9 +630,3 @@ export interface IBook2 extends TigrisCollectionType { title: string; metadata: object; } - -export interface Alert extends TigrisTopicType { - id: number; - name?: number; - text: string; -} diff --git a/src/__tests__/tigris.schema.spec.ts b/src/__tests__/tigris.schema.spec.ts index 7be4c12..ab77c83 100644 --- a/src/__tests__/tigris.schema.spec.ts +++ b/src/__tests__/tigris.schema.spec.ts @@ -1,4 +1,4 @@ -import {CollectionType, TigrisCollectionType, TigrisDataTypes, TigrisSchema,} from "../types"; +import { TigrisCollectionType, TigrisDataTypes, TigrisSchema,} from "../types"; import {Utility} from "../utility"; describe("schema tests", () => { @@ -34,7 +34,7 @@ describe("schema tests", () => { type: TigrisDataTypes.BYTE_STRING } }; - expect(Utility._toJSONSchema("basicCollection", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("basicCollection", schema)) .toBe(Utility._readTestDataFile("basicCollection.json")); }); @@ -54,7 +54,7 @@ describe("schema tests", () => { type: TigrisDataTypes.OBJECT } }; - expect(Utility._toJSONSchema("basicCollectionWithObjectType", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("basicCollectionWithObjectType", schema)) .toBe(Utility._readTestDataFile("basicCollectionWithObjectType.json")); }); @@ -92,7 +92,7 @@ describe("schema tests", () => { type: TigrisDataTypes.BYTE_STRING } }; - expect(Utility._toJSONSchema("multiplePKeys", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("multiplePKeys", schema)) .toBe(Utility._readTestDataFile("multiplePKeys.json")); }); @@ -119,7 +119,7 @@ describe("schema tests", () => { type: addressSchema } }; - expect(Utility._toJSONSchema("nestedCollection", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("nestedCollection", schema)) .toBe(Utility._readTestDataFile("nestedCollection.json")); }); @@ -138,7 +138,7 @@ describe("schema tests", () => { } } }; - expect(Utility._toJSONSchema("collectionWithPrimitiveArrays", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("collectionWithPrimitiveArrays", schema)) .toBe(Utility._readTestDataFile("collectionWithPrimitiveArrays.json")); }); @@ -168,7 +168,7 @@ describe("schema tests", () => { } } }; - expect(Utility._toJSONSchema("collectionWithObjectArrays", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("collectionWithObjectArrays", schema)) .toBe(Utility._readTestDataFile("collectionWithObjectArrays.json")); }); @@ -235,7 +235,7 @@ describe("schema tests", () => { } } }; - expect(Utility._toJSONSchema("multiLevelPrimitiveArray", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("multiLevelPrimitiveArray", schema)) .toBe(Utility._readTestDataFile("multiLevelPrimitiveArray.json")); }); @@ -313,7 +313,7 @@ describe("schema tests", () => { } } }; - expect(Utility._toJSONSchema("multiLevelObjectArray", CollectionType.DOCUMENTS, schema)) + expect(Utility._toJSONSchema("multiLevelObjectArray", schema)) .toBe(Utility._readTestDataFile("multiLevelObjectArray.json")); }); }); diff --git a/src/consumables/utils.ts b/src/consumables/utils.ts deleted file mode 100644 index deea24f..0000000 --- a/src/consumables/utils.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { ClientReadableStream } from "@grpc/grpc-js"; -import { Readable } from "node:stream"; - -function _next( - stream: ClientReadableStream, - transform: (arg: TResp) => T -): AsyncIterableIterator { - const iter: () => AsyncIterableIterator = async function* () { - for await (const message of stream) { - yield transform(message); - } - return; - }; - - return iter(); -} - -// Utility to convert grpc response streams to Readable streams -export function clientReadableToStream( - stream: ClientReadableStream, - transform: (arg: TResp) => T -): Readable { - return Readable.from(_next(stream, transform)); -} diff --git a/src/db.ts b/src/db.ts index 7ebe56f..a8f69e9 100644 --- a/src/db.ts +++ b/src/db.ts @@ -4,7 +4,6 @@ import { CollectionInfo, CollectionMetadata, CollectionOptions, - CollectionType, CommitTransactionResponse, DatabaseDescription, DatabaseMetadata, @@ -27,7 +26,6 @@ import { Collection } from "./collection"; import { Session } from "./session"; import { Utility } from "./utility"; import { Metadata, ServiceError } from "@grpc/grpc-js"; -import { Topic } from "./topic"; import { TigrisClientConfig } from "./tigris"; import { Log } from "./utils/logger"; @@ -55,32 +53,18 @@ export class DB { ): Promise> { return this.createOrUpdate( collectionName, - CollectionType.DOCUMENTS, schema, () => new Collection(collectionName, this._db, this.grpcClient, this.config) ); } - public createOrUpdateTopic( - topicName: string, - schema: TigrisSchema - ): Promise> { - return this.createOrUpdate( - topicName, - CollectionType.MESSAGES, - schema, - () => new Topic(topicName, this._db, this.grpcClient, this.config) - ); - } - private createOrUpdate( name: string, - type: CollectionType, schema: TigrisSchema, resolver: () => R ): Promise { return new Promise((resolve, reject) => { - const rawJSONSchema: string = Utility._toJSONSchema(name, type, schema); + const rawJSONSchema: string = Utility._toJSONSchema(name, schema); Log.debug(rawJSONSchema); const createOrUpdateCollectionRequest = new ProtoCreateOrUpdateCollectionRequest() .setDb(this._db) @@ -139,6 +123,19 @@ export class DB { }); } + public dropAllCollections(): Promise { + return new Promise((resolve, reject) => { + this.listCollections() + .then((value: Array) => { + for (const collectionInfo of value) { + this.dropCollection(collectionInfo.name); + } + resolve(); + }) + .catch((error) => reject(error)); + }); + } + public describe(): Promise { return new Promise((resolve, reject) => { this.grpcClient.describeDatabase( @@ -174,10 +171,6 @@ export class DB { return new Collection(collectionName, this.db, this.grpcClient, this.config); } - public getTopic(topicName: string): Topic { - return new Topic(topicName, this.db, this.grpcClient, this.config); - } - public transact(fn: (tx: Session) => void): Promise { return new Promise((resolve, reject) => { this.beginTransaction() diff --git a/src/index.ts b/src/index.ts index c9793ed..b8e42bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,4 +2,3 @@ export * from "./collection"; export * from "./db"; export * from "./session"; export * from "./tigris"; -export * from "./topic"; diff --git a/src/tigris.ts b/src/tigris.ts index c06fb75..023590f 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -6,8 +6,6 @@ import { ChannelCredentials, Metadata, status } from "@grpc/grpc-js"; import { CreateDatabaseRequest as ProtoCreateDatabaseRequest, DatabaseOptions as ProtoDatabaseOptions, - DropDatabaseRequest as ProtoDropDatabaseRequest, - ListDatabasesRequest as ProtoListDatabasesRequest, } from "./proto/server/v1/api_pb"; import { GetInfoRequest as ProtoGetInfoRequest } from "./proto/server/v1/observability_pb"; import { HealthCheckInput as ProtoHealthCheckInput } from "./proto/server/v1/health_pb"; @@ -15,13 +13,7 @@ import { HealthCheckInput as ProtoHealthCheckInput } from "./proto/server/v1/hea import path from "node:path"; import appRootPath from "app-root-path"; import * as dotenv from "dotenv"; -import { - DatabaseInfo, - DatabaseMetadata, - DatabaseOptions, - DropDatabaseResponse, - ServerMetadata, -} from "./types"; +import { DatabaseOptions, ServerMetadata } from "./types"; import { GetAccessTokenRequest as ProtoGetAccessTokenRequest, @@ -39,6 +31,7 @@ const AuthorizationBearer = "Bearer "; export interface TigrisClientConfig { serverUrl?: string; + projectName?: string; /** * Use clientId/clientSecret to authenticate production services. * Obtains at console.preview.tigrisdata.cloud in `Applications Keys` section @@ -152,7 +145,11 @@ export class Tigris { } if (config.serverUrl === undefined) { config.serverUrl = DEFAULT_URL; - + if (!("TIGRIS_PROJECT" in process.env)) { + throw new Error("Unable to resolve TIGRIS_PROJECT environment variable"); + } else { + config.projectName = process.env.TIGRIS_PROJECT; + } if ("TIGRIS_URI" in process.env) { config.serverUrl = process.env.TIGRIS_URI; } @@ -245,64 +242,8 @@ export class Tigris { Log.info(`Using Tigris at: ${config.serverUrl}`); } - /** - * Lists the databases - * @return {Promise>} a promise of an array of - * DatabaseInfo - */ - public listDatabases(): Promise> { - return new Promise>((resolve, reject) => { - this.grpcClient.listDatabases(new ProtoListDatabasesRequest(), (error, response) => { - if (error) { - reject(error); - } else { - const result = response - .getDatabasesList() - .map( - (protoDatabaseInfo) => - new DatabaseInfo(protoDatabaseInfo.getDb(), new DatabaseMetadata()) - ); - resolve(result); - } - }); - }); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public createDatabaseIfNotExists(db: string, _options?: DatabaseOptions): Promise { - return new Promise((resolve, reject) => { - this.grpcClient.createDatabase( - new ProtoCreateDatabaseRequest().setDb(db).setOptions(new ProtoDatabaseOptions()), - // eslint-disable-next-line @typescript-eslint/no-unused-vars - (error, _response) => { - if (error && error.code != status.ALREADY_EXISTS) { - reject(error); - } else { - resolve(new DB(db, this.grpcClient, this._config)); - } - } - ); - }); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public dropDatabase(db: string, _options?: DatabaseOptions): Promise { - return new Promise((resolve, reject) => { - this.grpcClient.dropDatabase( - new ProtoDropDatabaseRequest().setDb(db).setOptions(new ProtoDatabaseOptions()), - (error, response) => { - if (error) { - reject(error); - } else { - resolve(new DropDatabaseResponse(response.getStatus(), response.getMessage())); - } - } - ); - }); - } - - public getDatabase(db: string): DB { - return new DB(db, this.grpcClient, this._config); + public getDatabase(): DB { + return new DB(this._config.projectName, this.grpcClient, this._config); } public getServerMetadata(): Promise { @@ -354,4 +295,21 @@ export class Tigris { clearInterval(this.pingId); } } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + private createDatabaseIfNotExists(db: string, _options?: DatabaseOptions): Promise { + return new Promise((resolve, reject) => { + this.grpcClient.createDatabase( + new ProtoCreateDatabaseRequest().setDb(db).setOptions(new ProtoDatabaseOptions()), + // eslint-disable-next-line @typescript-eslint/no-unused-vars + (error, _response) => { + if (error && error.code != status.ALREADY_EXISTS) { + reject(error); + } else { + resolve(new DB(db, this.grpcClient, this._config)); + } + } + ); + }); + } } diff --git a/src/topic.ts b/src/topic.ts deleted file mode 100644 index a41ce43..0000000 --- a/src/topic.ts +++ /dev/null @@ -1,309 +0,0 @@ -import * as grpc from "@grpc/grpc-js"; -import { TigrisClient } from "./proto/server/v1/api_grpc_pb"; -import * as server_v1_api_pb from "./proto/server/v1/api_pb"; -import { - PublishRequest as ProtoPublishRequest, - PublishRequestOptions as ProtoPublishRequestOptions, - SubscribeRequest as ProtoSubscribeRequest, - SubscribeRequestOptions as ProtoSubscribeRequestOptions, - SubscribeResponse as ProtoSubscribeResponse, -} from "./proto/server/v1/api_pb"; -import { Filter, PublishOptions, SubscribeOptions, TigrisTopicType } from "./types"; -import { Utility } from "./utility"; -import { TigrisClientConfig } from "./tigris"; -import { Readable } from "node:stream"; -import { clientReadableToStream } from "./consumables/utils"; -import { ReadOnlyCollection } from "./collection"; - -/** - * Callback to receive events for a topic from server - */ -export interface SubscribeCallback { - /** - * Receives a message from server. Can be called many times but is never called after - * {@link onError} or {@link onEnd} are called. - * - * @param message - */ - onNext(message: T): void; - - /** - * Receives a notification of successful stream completion. - * - *

May only be called once and if called it must be the last method called. In particular, - * if an exception is thrown by an implementation of {@link onEnd} no further calls to any - * method are allowed. - */ - onEnd(): void; - - /** - * Receives terminating error from the stream. - * @param err - */ - onError(err: Error): void; -} - -/** - * The **Topic** class represents a events stream in Tigris. - */ -export class Topic extends ReadOnlyCollection { - private readonly _topicName: string; - - constructor(topicName: string, db: string, grpcClient: TigrisClient, config: TigrisClientConfig) { - super(topicName, db, grpcClient, config); - this._topicName = topicName; - } - - /** - * Name of this topic - */ - get topicName(): string { - return this._topicName; - } - - /** - * Publish multiple events to the topic - * - * @param messages - Array of events to publish - * @param {PublishOptions} options - Optional publishing options - * - * @example Publish messages to topic - *``` - * const tigris = new Tigris(config); - * const topic = tigris.getDatabase("my_db").getTopic("my_topic"); - * const messages = [new Message(1), new Message(2)]; - * topic.publishMany(messages) - * .then(result => console.log(result)) - * .catch(err => console.log(err)); - * ``` - * @returns Promise of published messages - */ - publishMany(messages: Array, options?: PublishOptions): Promise> { - return new Promise>((resolve, reject) => { - const messagesUintArray = new Array(); - const textEncoder = new TextEncoder(); - for (const message of messages) { - messagesUintArray.push(textEncoder.encode(Utility.objToJsonString(message))); - } - - const protoRequest = new ProtoPublishRequest() - .setDb(this.db) - .setCollection(this._topicName) - .setMessagesList(messagesUintArray); - - if (options) { - protoRequest.setOptions(new ProtoPublishRequestOptions().setPartition(options.partition)); - } - - this.grpcClient.publish( - protoRequest, - (error: grpc.ServiceError, response: server_v1_api_pb.PublishResponse): void => { - if (error !== undefined && error !== null) { - reject(error); - } else { - let messageIndex = 0; - const clonedMessages: T[] = Object.assign([], messages); - - for (const value of response.getKeysList_asU8()) { - const keyValueJsonObj: object = Utility.jsonStringToObj( - Utility.uint8ArrayToString(value), - this.config - ); - for (const fieldName of Object.keys(keyValueJsonObj)) { - Reflect.set(clonedMessages[messageIndex], fieldName, keyValueJsonObj[fieldName]); - messageIndex++; - } - } - resolve(clonedMessages); - } - } - ); - }); - } - - /** - * Publish a single message to topic - * - * @example Publish a message to topic - *``` - * const tigris = new Tigris(config); - * const topic = tigris.getDatabase("my_db").getTopic("my_topic"); - * topic.publish(new Message(1)) - * .then(result => console.log(result)) - * .catch(err => console.log(err)); - *``` - - * @param message - Message to publish - * @param {PublishOptions} options - Optional publishing options - * - * @returns Promise of the published message - */ - publish(message: T, options?: PublishOptions): Promise { - return new Promise((resolve, reject) => { - const messageArr: Array = new Array(); - messageArr.push(message); - this.publishMany(messageArr, options) - .then((messages) => { - resolve(messages[0]); - }) - .catch((error) => { - reject(error); - }); - }); - } - - /** - * Subscribe to listen for messages in a topic. Users can consume messages in one of two ways: - * 1. By providing an optional {@link SubscribeCallback} as param - * 2. By consuming {@link Readable} stream when no callback is provided - * - * @example Subscribe using callback - *``` - * const tigris = new Tigris(config); - * const topic = tigris.getDatabase("my_db").getTopic("my_topic"); - * - * topic.subscribe({ - * onNext(message: T) { - * console.log(message); - * }, - * onError(err: Error) { - * console.log(err); - * }, - * onEnd() { - * console.log("All messages consumed"); - * } - * }); - *``` - * - * @example Subscribe using {@link Readable} stream if callback is omitted - *``` - * const tigris = new Tigris(config); - * const topic = tigris.getDatabase("my_db").getTopic("my_topic"); - * const stream = topic.subscribe() as Readable; - * - * stream.on("data", (message: T) => console.log(message)); - * stream.on("error", (err: Error) => console.log(err)); - * stream.on("end", () => console.log("All messages consumed")); - *``` - * - * @param {SubscribeCallback} callback - Optional callback to consume messages - * @param {SubscribeOptions} options - Optional subscription options - * - * @returns {Readable} if no callback is provided, else nothing is returned - */ - subscribe(callback?: SubscribeCallback, options?: SubscribeOptions): Readable | void { - return this.subscribeWithFilter(undefined, callback, options); - } - - /** - * Subscribe to listen for messages in a topic that match given filter. Users can consume - * messages in one of two ways: - * 1. By providing an optional {@link SubscribeCallback} as param - * 2. By consuming {@link Readable} stream when no callback is provided - * - * @example Subscribe using callback - *``` - * const tigris = new Tigris(config); - * const topic = tigris.getDatabase("my_db").getTopic("my_topic"); - * const balanceLessThanThreshold = { - * op: SelectorFilterOperator.LT, - * fields: { - * balance: 200 - * } - * }; - * - * topic.subscribeWithFilter( - * balanceLessThanThreshold, - * { - * onNext(message: T) { - * console.log(message); - * }, - * onError(err: Error) { - * console.log(err); - * }, - * onEnd() { - * console.log("All messages consumed"); - * } - * } - * ); - *``` - * - * @example Subscribe using {@link Readable} stream if callback is omitted - *``` - * const tigris = new Tigris(config); - * const topic = tigris.getDatabase("my_db").getTopic("my_topic"); - * const balanceLessThanThreshold = { - * op: SelectorFilterOperator.LT, - * fields: { - * balance: 200 - * } - * }; - * const stream = topic.subscribe(balanceLessThanThreshold) as Readable; - * - * stream.on("data", (message: T) => console.log(message)); - * stream.on("error", (err: Error) => console.log(err)); - * stream.on("end", () => console.log("All messages consumed")); - *``` - * - * @param {Filter} filter - Subscription will only return messages that match this query - * @param {SubscribeCallback} callback - Optional callback to consume messages - * @param {SubscribeOptions} options - Optional subscription options - * - * @returns {Readable} if no callback is provided, else nothing is returned - */ - subscribeWithFilter( - filter: Filter, - callback?: SubscribeCallback, - options?: SubscribeOptions - ): Readable | void { - const subscribeRequest = new ProtoSubscribeRequest() - .setDb(this.db) - .setCollection(this._topicName); - - if (filter !== undefined) { - subscribeRequest.setFilter(Utility.stringToUint8Array(Utility.filterToString(filter))); - } - - if (options) { - subscribeRequest.setOptions( - new ProtoSubscribeRequestOptions().setPartitionsList(options.partitions) - ); - } - - const transform: (arg: ProtoSubscribeResponse) => T = (resp: ProtoSubscribeResponse) => { - return Utility.jsonStringToObj( - Utility._base64Decode(resp.getMessage_asB64()), - this.config - ); - }; - - const stream: grpc.ClientReadableStream = - this.grpcClient.subscribe(subscribeRequest); - - if (callback !== undefined) { - stream.on("data", (subscribeResponse: ProtoSubscribeResponse) => { - callback.onNext(transform(subscribeResponse)); - }); - - stream.on("error", (error) => callback.onError(error)); - stream.on("end", () => callback.onEnd()); - } else { - return clientReadableToStream(stream, transform); - } - } - - subscribeToPartitions( - partitions: Array, - callback?: SubscribeCallback - ): Readable | void { - return this.subscribeWithFilterToPartitions(undefined, partitions, callback); - } - - subscribeWithFilterToPartitions( - filter: Filter, - partitions: Array, - callback?: SubscribeCallback - ): Readable | void { - return this.subscribeWithFilter(filter, callback, new SubscribeOptions(partitions)); - } -} diff --git a/src/types.ts b/src/types.ts index 4803d8d..144ebbd 100644 --- a/src/types.ts +++ b/src/types.ts @@ -403,15 +403,6 @@ export class ServerMetadata { // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface TigrisCollectionType {} -// Marker interface -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface TigrisTopicType extends TigrisCollectionType {} - -export enum CollectionType { - DOCUMENTS = "documents", - MESSAGES = "messages", -} - export enum LogicalOperator { AND = "$and", OR = "$or", @@ -482,14 +473,6 @@ export type TigrisSchema = { }; }; -export type TigrisTopicSchema = { - [K in keyof T]: { - type: TigrisDataTypes | TigrisTopicSchema; - key?: TigrisPartitionKey; - items?: TigrisArrayItem; - }; -}; - export type TigrisArrayItem = { type: TigrisDataTypes | TigrisSchema; items?: TigrisArrayItem | TigrisDataTypes; diff --git a/src/utility.ts b/src/utility.ts index 8a57fc2..a4ba13d 100644 --- a/src/utility.ts +++ b/src/utility.ts @@ -3,7 +3,6 @@ import json_bigint from "json-bigint"; import { Session } from "./session"; import { - CollectionType, DeleteRequestOptions, LogicalFilter, LogicalOperator, @@ -16,7 +15,6 @@ import { TigrisCollectionType, TigrisDataTypes, TigrisSchema, - TigrisTopicSchema, UpdateFields, UpdateFieldsOperator, UpdateRequestOptions, @@ -239,11 +237,7 @@ export const Utility = { return toReturn; }, - _toJSONSchema( - collectionName: string, - collectionType: CollectionType, - schema: TigrisSchema | TigrisTopicSchema - ): string { + _toJSONSchema(collectionName: string, schema: TigrisSchema): string { const root = {}; const pkeyMap = {}; const keyMap = {}; @@ -251,12 +245,8 @@ export const Utility = { root["additionalProperties"] = false; root["type"] = "object"; root["properties"] = this._getSchemaProperties(schema, pkeyMap, keyMap); - root["collection_type"] = collectionType; - if (collectionType === "documents") { - Utility._postProcessDocumentSchema(root, pkeyMap); - } else if (collectionType === "messages") { - Utility._postProcessMessageSchema(root, keyMap); - } + root["collection_type"] = "documents"; + Utility._postProcessDocumentSchema(root, pkeyMap); return Utility.objToJsonString(root); }, /* @@ -295,11 +285,7 @@ export const Utility = { return result; }, - _getSchemaProperties( - schema: TigrisSchema | TigrisTopicSchema, - pkeyMap: object, - keyMap: object - ): object { + _getSchemaProperties(schema: TigrisSchema, pkeyMap: object, keyMap: object): object { const properties = {}; for (const property of Object.keys(schema)) { From 80d838a9908715876d16ccae2a9d43d0504345b6 Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Mon, 21 Nov 2022 18:41:00 -0800 Subject: [PATCH 02/15] Remove collection type (#173) fix: Removed explicit collection_type=documents --- src/__tests__/data/basicCollection.json | 1 - src/__tests__/data/basicCollectionWithObjectType.json | 1 - src/__tests__/data/collectionWithObjectArrays.json | 1 - src/__tests__/data/collectionWithPrimitiveArrays.json | 1 - src/__tests__/data/multiLevelObjectArray.json | 1 - src/__tests__/data/multiLevelPrimitiveArray.json | 1 - src/__tests__/data/multiplePKeys.json | 1 - src/__tests__/data/nestedCollection.json | 1 - src/utility.ts | 1 - 9 files changed, 9 deletions(-) diff --git a/src/__tests__/data/basicCollection.json b/src/__tests__/data/basicCollection.json index cd0a37c..1ceafa6 100644 --- a/src/__tests__/data/basicCollection.json +++ b/src/__tests__/data/basicCollection.json @@ -35,7 +35,6 @@ "format": "byte" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/__tests__/data/basicCollectionWithObjectType.json b/src/__tests__/data/basicCollectionWithObjectType.json index 60810a3..5f59723 100644 --- a/src/__tests__/data/basicCollectionWithObjectType.json +++ b/src/__tests__/data/basicCollectionWithObjectType.json @@ -15,7 +15,6 @@ "type": "object" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/__tests__/data/collectionWithObjectArrays.json b/src/__tests__/data/collectionWithObjectArrays.json index f63d30e..973e7d0 100644 --- a/src/__tests__/data/collectionWithObjectArrays.json +++ b/src/__tests__/data/collectionWithObjectArrays.json @@ -31,7 +31,6 @@ "format": "uuid" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/__tests__/data/collectionWithPrimitiveArrays.json b/src/__tests__/data/collectionWithPrimitiveArrays.json index cb64066..913944b 100644 --- a/src/__tests__/data/collectionWithPrimitiveArrays.json +++ b/src/__tests__/data/collectionWithPrimitiveArrays.json @@ -20,7 +20,6 @@ "format": "uuid" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/__tests__/data/multiLevelObjectArray.json b/src/__tests__/data/multiLevelObjectArray.json index 8ede319..944ef91 100644 --- a/src/__tests__/data/multiLevelObjectArray.json +++ b/src/__tests__/data/multiLevelObjectArray.json @@ -123,7 +123,6 @@ "format": "uuid" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/__tests__/data/multiLevelPrimitiveArray.json b/src/__tests__/data/multiLevelPrimitiveArray.json index ca00ec6..6b6d0fa 100644 --- a/src/__tests__/data/multiLevelPrimitiveArray.json +++ b/src/__tests__/data/multiLevelPrimitiveArray.json @@ -68,7 +68,6 @@ "format": "uuid" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/__tests__/data/multiplePKeys.json b/src/__tests__/data/multiplePKeys.json index 9247138..a5406a6 100644 --- a/src/__tests__/data/multiplePKeys.json +++ b/src/__tests__/data/multiplePKeys.json @@ -35,7 +35,6 @@ "format": "byte" } }, - "collection_type": "documents", "primary_key": [ "uuid", "id" diff --git a/src/__tests__/data/nestedCollection.json b/src/__tests__/data/nestedCollection.json index 5a045e1..a708d22 100644 --- a/src/__tests__/data/nestedCollection.json +++ b/src/__tests__/data/nestedCollection.json @@ -28,7 +28,6 @@ "format": "uuid" } }, - "collection_type": "documents", "primary_key": [ "id" ] diff --git a/src/utility.ts b/src/utility.ts index a4ba13d..bdf3d2f 100644 --- a/src/utility.ts +++ b/src/utility.ts @@ -245,7 +245,6 @@ export const Utility = { root["additionalProperties"] = false; root["type"] = "object"; root["properties"] = this._getSchemaProperties(schema, pkeyMap, keyMap); - root["collection_type"] = "documents"; Utility._postProcessDocumentSchema(root, pkeyMap); return Utility.objToJsonString(root); }, From b881028601187dd402010a0635d48fda784d73a4 Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Mon, 21 Nov 2022 21:58:37 -0800 Subject: [PATCH 03/15] Cleanup event streaming (#174) fix: Cleaned up event streaming related code --- src/collection.ts | 29 -------------------- src/types.ts | 68 ----------------------------------------------- 2 files changed, 97 deletions(-) diff --git a/src/collection.ts b/src/collection.ts index 0f32160..144d247 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -19,7 +19,6 @@ import { ReadRequestOptions, SelectorFilterOperator, SimpleUpdateField, - StreamEvent, TigrisCollectionType, UpdateFields, UpdateRequestOptions, @@ -30,34 +29,6 @@ import { SearchRequest, SearchRequestOptions, SearchResult } from "./search/type import { TigrisClientConfig } from "./tigris"; import { Cursor, ReadCursorInitializer } from "./consumables/cursor"; -/** - * Callback to receive events from server - */ -export interface EventsCallback { - /** - * Receives a message from server. Can be called many times but is never called after - * {@link onError} or {@link onEnd} are called. - * - * @param event - */ - onNext(event: StreamEvent): void; - - /** - * Receives a notification of successful stream completion. - * - *

May only be called once and if called it must be the last method called. In particular, - * if an exception is thrown by an implementation of {@link onEnd} no further calls to any - * method are allowed. - */ - onEnd(): void; - - /** - * Receives terminating error from the stream. - * @param error - */ - onError(error: Error): void; -} - interface ICollection { readonly collectionName: string; readonly db: string; diff --git a/src/types.ts b/src/types.ts index 144ebbd..9a98c86 100644 --- a/src/types.ts +++ b/src/types.ts @@ -301,42 +301,6 @@ export class ReadRequestOptions { export class TransactionOptions {} -export class StreamEvent { - private readonly _txId: string; - private readonly _collection: string; - private readonly _op: string; - private readonly _data: T; - private readonly _last: boolean; - - constructor(txId: string, collection: string, op: string, data: T, last: boolean) { - this._txId = txId; - this._collection = collection; - this._op = op; - this._data = data; - this._last = last; - } - - get txId(): string { - return this._txId; - } - - get collection(): string { - return this._collection; - } - - get op(): string { - return this._op; - } - - get data(): T { - return this._data; - } - - get last(): boolean { - return this._last; - } -} - export class CommitTransactionResponse extends TigrisResponse { constructor(status: string) { super(status); @@ -355,38 +319,6 @@ export class TransactionResponse extends TigrisResponse { } } -export class PublishOptions { - private _partition: number; - - constructor(partition: number) { - this._partition = partition; - } - - get partition(): number { - return this._partition; - } - - set partition(value: number) { - this._partition = value; - } -} - -export class SubscribeOptions { - private _partitions: Array; - - constructor(partitions: Array) { - this._partitions = partitions; - } - - get partitions(): Array { - return this._partitions; - } - - set partitions(value: Array) { - this._partitions = value; - } -} - export class ServerMetadata { private readonly _serverVersion: string; From d9891d3ffb9d4efdcfc60a42f9a9c8f0e88c339f Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Tue, 22 Nov 2022 13:29:36 -0800 Subject: [PATCH 04/15] Fix setting document metadata from the response --- src/__tests__/test-service.ts | 2 ++ src/__tests__/tigris.rpc.spec.ts | 59 ++++++++++++++++++++++++++++++++ src/collection.ts | 47 ++++++++++++------------- 3 files changed, 82 insertions(+), 26 deletions(-) diff --git a/src/__tests__/test-service.ts b/src/__tests__/test-service.ts index 7be4233..7eb5abd 100644 --- a/src/__tests__/test-service.ts +++ b/src/__tests__/test-service.ts @@ -273,6 +273,8 @@ export class TestTigrisService { if (call.request.getCollection() === "books-with-optional-field") { const extractedKeyFromAuthor: number = JSON.parse(Utility._base64Decode(call.request.getDocumentsList_asB64()[i - 1]))["author"]; keyList.push(Utility._base64Encode("{\"id\":" + extractedKeyFromAuthor + "}")); + } else if (call.request.getCollection() === "books-multi-pk") { + keyList.push(Utility._base64Encode("{\"id\":" + i + ", \"id2\":" + i+1 + "}")); } else { keyList.push(Utility._base64Encode("{\"id\":" + i + "}")); } diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index 85909cb..2a71daa 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -158,6 +158,58 @@ describe("rpc tests", () => { return insertionPromise; }); + it("insert_multi_pk", () => { + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); + const insertionPromise = db1.getCollection("books-multi-pk").insertOne({ + id: 0, + id2: 0, + title: "science book", + metadata: { + publish_date: new Date(), + num_pages: 100, + } + }); + insertionPromise.then(insertedBook => { + expect(insertedBook.id).toBe(1); + expect(insertedBook.id2).toBe(11); + }); + return insertionPromise; + }); + + it("insert_multi_pk_many", () => { + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); + const db1 = tigris.getDatabase(); + const insertionPromise = db1.getCollection("books-multi-pk").insertMany([ + { + id: 0, + id2: 0, + title: "science book", + metadata: { + publish_date: new Date(), + num_pages: 100, + } + }, + { + id: 0, + id2: 0, + title: "science book", + metadata: { + publish_date: new Date(), + num_pages: 100, + } + } + ]); + insertionPromise.then(insertedBook => { + expect(insertedBook.length).toBe(2); + expect(insertedBook[0].id).toBe(1); + expect(insertedBook[0].id2).toBe(11); + expect(insertedBook[1].id).toBe(2); + expect(insertedBook[1].id2).toBe(21); + }); + return insertionPromise; + }); + it("insertWithOptionalField", () => { const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); const db1 = tigris.getDatabase(); @@ -630,3 +682,10 @@ export interface IBook2 extends TigrisCollectionType { title: string; metadata: object; } + +export interface IBookMPK extends TigrisCollectionType { + id?: number; + id2?: number; + title: string; + metadata: object; +} diff --git a/src/collection.ts b/src/collection.ts index 144d247..4d117b1 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -184,7 +184,7 @@ export abstract class ReadOnlyCollection impleme /** * The **Collection** class represents Tigris collection allowing insert/find/update/delete/search - * and events operations. + * operations. */ export class Collection extends ReadOnlyCollection { constructor( @@ -196,6 +196,24 @@ export class Collection extends ReadOnlyCollecti super(collectionName, db, grpcClient, config); } + private setDocsMetadata(docs: Array, keys: Array): Array { + let docIndex = 0; + const clonedDocs: T[] = Object.assign([], docs); + + for (const value of keys) { + const keyValueJsonObj: object = Utility.jsonStringToObj( + Utility.uint8ArrayToString(value), + this.config + ); + for (const fieldName of Object.keys(keyValueJsonObj)) { + Reflect.set(clonedDocs[docIndex], fieldName, keyValueJsonObj[fieldName]); + } + docIndex++; + } + + return clonedDocs; + } + /** * Inserts multiple documents in Tigris collection. * @@ -221,19 +239,7 @@ export class Collection extends ReadOnlyCollecti if (error !== undefined && error !== null) { reject(error); } else { - let docIndex = 0; - const clonedDocs: T[] = Object.assign([], docs); - - for (const value of response.getKeysList_asU8()) { - const keyValueJsonObj: object = Utility.jsonStringToObj( - Utility.uint8ArrayToString(value), - this.config - ); - for (const fieldName of Object.keys(keyValueJsonObj)) { - Reflect.set(clonedDocs[docIndex], fieldName, keyValueJsonObj[fieldName]); - docIndex++; - } - } + const clonedDocs = this.setDocsMetadata(docs, response.getKeysList_asU8()); resolve(clonedDocs); } } @@ -285,18 +291,7 @@ export class Collection extends ReadOnlyCollecti if (error !== undefined && error !== null) { reject(error); } else { - let docIndex = 0; - const clonedDocs: T[] = Object.assign([], docs); - for (const value of response.getKeysList_asU8()) { - const keyValueJsonObj: object = Utility.jsonStringToObj( - Utility.uint8ArrayToString(value), - this.config - ); - for (const fieldName of Object.keys(keyValueJsonObj)) { - Reflect.set(clonedDocs[docIndex], fieldName, keyValueJsonObj[fieldName]); - docIndex++; - } - } + const clonedDocs = this.setDocsMetadata(docs, response.getKeysList_asU8()); resolve(clonedDocs); } } From 7ed6e6730860f1cf418c1dfe5219129666005090 Mon Sep 17 00:00:00 2001 From: Ovais Tariq Date: Wed, 30 Nov 2022 16:29:15 -0800 Subject: [PATCH 05/15] Register schema now requires database name to be passed. (#176) Automatic schema management now expects collections for only a single database. --- api/proto | 2 +- src/__tests__/utils/manifest-loader.spec.ts | 98 ++++++++++++--------- src/tigris.ts | 31 +++---- src/utils/manifest-loader.ts | 94 +++++++++----------- 4 files changed, 115 insertions(+), 110 deletions(-) diff --git a/api/proto b/api/proto index 8699c98..e28162d 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit 8699c9865c135f40850e7b17c734831758962763 +Subproject commit e28162d35c3ffa0ce20a3dd3675bf4e34b3f2711 diff --git a/src/__tests__/utils/manifest-loader.spec.ts b/src/__tests__/utils/manifest-loader.spec.ts index 16fea17..6438c4f 100644 --- a/src/__tests__/utils/manifest-loader.spec.ts +++ b/src/__tests__/utils/manifest-loader.spec.ts @@ -1,16 +1,19 @@ -import { canBeSchema, loadTigrisManifest, TigrisManifest } from "../../utils/manifest-loader"; +import { + canBeSchema, + DatabaseManifest, + loadTigrisManifest, +} from "../../utils/manifest-loader"; import { TigrisDataTypes } from "../../types"; import { TigrisFileNotFoundError, TigrisMoreThanOneSchemaDefined } from "../../error"; describe("Manifest loader", () => { - it("generates manifest from file system", () => { - const schemaPath = process.cwd() + "/src/__tests__/data/models"; - const manifest: TigrisManifest = loadTigrisManifest(schemaPath); - expect(manifest).toHaveLength(3); + it("generates manifest from directory with single collection", () => { + const schemaPath = process.cwd() + "/src/__tests__/data/models/catalog"; + const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); + expect(dbManifest.collections).toHaveLength(1); - const expected: TigrisManifest = [{ - "dbName": "catalog", + const expected: DatabaseManifest = { "collections": [{ "collectionName": "products", "schema": { @@ -21,48 +24,63 @@ describe("Manifest loader", () => { }, "schemaName": "ProductSchema" }] - }, - { "dbName": "embedded", - "collections": [{ - "collectionName": "users", - "schemaName": "userSchema", - "schema": { - "created": { "type": "date-time" }, - "email": { "type": "string" }, - "identities": { - "type": "array", - "items": { - "type": { - "connection": { "type": "string" }, - "isSocial": { "type": "boolean" }, - "provider": { "type": "string" }, - "user_id": { "type": "string" } - } - } - }, - "name": { "type": "string" }, - "picture": { "type": "string" }, - "stats": { + }; + expect(dbManifest).toStrictEqual(expected); + }); + + it("generates manifest from directory with embedded data model", () => { + const schemaPath = process.cwd() + "/src/__tests__/data/models/embedded"; + const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); + expect(dbManifest.collections).toHaveLength(1); + + const expected: DatabaseManifest = { + "collections": [{ + "collectionName": "users", + "schemaName": "userSchema", + "schema": { + "created": { "type": "date-time" }, + "email": { "type": "string" }, + "identities": { + "type": "array", + "items": { "type": { - "loginsCount": { "type": "int64" } + "connection": { "type": "string" }, + "isSocial": { "type": "boolean" }, + "provider": { "type": "string" }, + "user_id": { "type": "string" } } - }, - "updated": { "type": "date-time" }, - "user_id": { "type": "string", "primary_key": { "order": 1 } } - } - }]}, - { "dbName": "empty", "collections": [] }, - ]; - expect(manifest).toStrictEqual(expected); + } + }, + "name": { "type": "string" }, + "picture": { "type": "string" }, + "stats": { + "type": { + "loginsCount": { "type": "int64" } + } + }, + "updated": { "type": "date-time" }, + "user_id": { "type": "string", "primary_key": { "order": 1 } } + } + }]}; + expect(dbManifest).toStrictEqual(expected); + }); + + it("does not generate manifest from empty directory", () => { + const schemaPath = process.cwd() + "/src/__tests__/data/models/empty"; + const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); + expect(dbManifest.collections).toHaveLength(0); + + const expected: DatabaseManifest = { "collections": [] }; + expect(dbManifest).toStrictEqual(expected); }); it("throws error for invalid path", () => { - const schemaPath = "/src/__tests__/data/models"; + const schemaPath = "/src/__tests__/data/doesNotExist"; expect(() => loadTigrisManifest(schemaPath)).toThrow(TigrisFileNotFoundError); }); it("throws error for multiple schema exports", () => { - const schemaPath = process.cwd() + "/src/__tests__/data/invalidModels"; + const schemaPath = process.cwd() + "/src/__tests__/data/invalidModels/multiExport"; expect(() => loadTigrisManifest(schemaPath)).toThrow(TigrisMoreThanOneSchemaDefined); }); diff --git a/src/tigris.ts b/src/tigris.ts index 023590f..1bb6a4b 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -23,7 +23,7 @@ import { import { DB } from "./db"; import { AuthClient } from "./proto/server/v1/auth_grpc_pb"; import { Utility } from "./utility"; -import { loadTigrisManifest, TigrisManifest } from "./utils/manifest-loader"; +import { loadTigrisManifest, DatabaseManifest } from "./utils/manifest-loader"; import { Log } from "./utils/logger"; const AuthorizationHeaderName = "authorization"; @@ -265,28 +265,25 @@ export class Tigris { * @param schemaPath - Directory location in file system. Recommended to * provide an absolute path, else loader will try to access application's root * path which may not be accurate. + * + * @param dbName - The name of the database to create the collections in. */ - public async registerSchemas(schemaPath: string) { + public async registerSchemas(schemaPath: string, dbName: string) { if (!path.isAbsolute(schemaPath)) { schemaPath = path.join(appRootPath.toString(), schemaPath); } - const manifest: TigrisManifest = loadTigrisManifest(schemaPath); - for (const dbManifest of manifest) { - // create DB - const tigrisDb = await this.createDatabaseIfNotExists(dbManifest.dbName); - Log.event(`Created database: ${dbManifest.dbName}`); + // create DB + const tigrisDb = await this.createDatabaseIfNotExists(dbName); + Log.event(`Created database: ${dbName}`); - for (const coll of dbManifest.collections) { - // Create a collection - const collection = await tigrisDb.createOrUpdateCollection( - coll.collectionName, - coll.schema - ); - Log.event( - `Created collection: ${collection.collectionName} from schema: ${coll.schemaName} in db: ${dbManifest.dbName}` - ); - } + const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); + for (const coll of dbManifest.collections) { + // Create a collection + const collection = await tigrisDb.createOrUpdateCollection(coll.collectionName, coll.schema); + Log.event( + `Created collection: ${collection.collectionName} from schema: ${coll.schemaName} in db: ${dbName}` + ); } } diff --git a/src/utils/manifest-loader.ts b/src/utils/manifest-loader.ts index 0c81db3..7ea2279 100644 --- a/src/utils/manifest-loader.ts +++ b/src/utils/manifest-loader.ts @@ -12,15 +12,12 @@ type CollectionManifest = { schema: TigrisSchema; }; -type DatabaseManifest = { - dbName: string; - collections: Array; -}; - /** - * Array of databases and collections in each database + * Array of collections in the database */ -export type TigrisManifest = Array; +export type DatabaseManifest = { + collections: Array; +}; /** * Loads the databases and schema definitions from file system that can be used @@ -28,7 +25,7 @@ export type TigrisManifest = Array; * * @return TigrisManifest */ -export function loadTigrisManifest(schemasPath: string): TigrisManifest { +export function loadTigrisManifest(schemasPath: string): DatabaseManifest { Log.event(`Scanning ${schemasPath} for Tigris schema definitions`); if (!fs.existsSync(schemasPath)) { @@ -36,59 +33,52 @@ export function loadTigrisManifest(schemasPath: string): TigrisManifest { throw new TigrisFileNotFoundError(`Directory not found: ${schemasPath}`); } - const tigrisFileManifest: TigrisManifest = new Array(); + const dbManifest: DatabaseManifest = { + collections: new Array(), + }; // load manifest from file structure - for (const schemaPathEntry of fs.readdirSync(schemasPath)) { - const dbDirPath = path.join(schemasPath, schemaPathEntry); - if (fs.lstatSync(dbDirPath).isDirectory()) { - Log.info(`Found DB definition ${schemaPathEntry}`); - const dbManifest: DatabaseManifest = { - dbName: schemaPathEntry, - collections: new Array(), - }; + for (const colsFileName of fs.readdirSync(schemasPath)) { + const collFilePath = path.join(schemasPath, colsFileName); + if (collFilePath.endsWith(COLL_FILE_SUFFIX) && fs.lstatSync(collFilePath).isFile()) { + Log.info(`Found Schema file ${colsFileName} in ${schemasPath}`); + const collName = colsFileName.slice( + 0, + Math.max(0, colsFileName.length - COLL_FILE_SUFFIX.length) + ); + + // eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module + const schemaFile = require(collFilePath); + const detectedSchemas = new Map>(); - for (const dbPathEntry of fs.readdirSync(dbDirPath)) { - if (dbPathEntry.endsWith(COLL_FILE_SUFFIX)) { - const collFilePath = path.join(dbDirPath, dbPathEntry); - if (fs.lstatSync(collFilePath).isFile()) { - Log.info(`Found Schema file ${dbPathEntry} in ${schemaPathEntry}`); - const collName = dbPathEntry.slice( - 0, - Math.max(0, dbPathEntry.length - COLL_FILE_SUFFIX.length) - ); - // eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module - const schemaFile = require(collFilePath); - const detectedSchemas = new Map>(); - // read schemas in that file - for (const [key, value] of Object.entries(schemaFile)) { - if (canBeSchema(value)) { - detectedSchemas.set(key, value as TigrisSchema); - } - } - if (detectedSchemas.size > 1) { - throw new TigrisMoreThanOneSchemaDefined(dbPathEntry, detectedSchemas.size); - } - for (const [name, def] of detectedSchemas) { - dbManifest.collections.push({ - collectionName: collName, - schema: def, - schemaName: name, - }); - Log.info(`Found schema definition: ${name}`); - } - } + // read schemas in that file + for (const [key, value] of Object.entries(schemaFile)) { + if (canBeSchema(value)) { + detectedSchemas.set(key, value as TigrisSchema); } } - if (dbManifest.collections.length === 0) { - Log.warn(`No valid schema definition found in ${schemaPathEntry}`); + + if (detectedSchemas.size > 1) { + throw new TigrisMoreThanOneSchemaDefined(collFilePath, detectedSchemas.size); + } + + for (const [name, def] of detectedSchemas) { + dbManifest.collections.push({ + collectionName: collName, + schema: def, + schemaName: name, + }); + Log.info(`Found schema definition: ${name}`); } - tigrisFileManifest.push(dbManifest); } } - Log.debug(`Generated Tigris Manifest: ${JSON.stringify(tigrisFileManifest)}`); - return tigrisFileManifest; + if (dbManifest.collections.length === 0) { + Log.warn(`No valid schema definition found in ${schemasPath}`); + } + + Log.debug(`Generated DB Manifest: ${JSON.stringify(dbManifest)}`); + return dbManifest; } /** From 46aeb3ee02ab625024c39844fca804167ad1ebcb Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Wed, 30 Nov 2022 19:17:32 -0800 Subject: [PATCH 06/15] fix: Fix empty URL setting Client fails to start if variable is defined but empty string. --- src/tigris.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tigris.ts b/src/tigris.ts index 1bb6a4b..b343e0a 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -150,10 +150,10 @@ export class Tigris { } else { config.projectName = process.env.TIGRIS_PROJECT; } - if ("TIGRIS_URI" in process.env) { + if (process.env.TIGRIS_URI?.trim().length > 0) { config.serverUrl = process.env.TIGRIS_URI; } - if ("TIGRIS_URL" in process.env) { + if (process.env.TIGRIS_URL?.trim().length > 0) { config.serverUrl = process.env.TIGRIS_URL; } } From dfe9cd4b830713db9bdc3a3af3e9de9e6912cc2f Mon Sep 17 00:00:00 2001 From: Ovais Tariq Date: Thu, 1 Dec 2022 19:57:30 -0800 Subject: [PATCH 07/15] Use the project name as DB name in registerSchema (#179) --- package.json | 3 ++- src/tigris.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a5edc76..c4b3f60 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "scripts": { "clean": "rm -rf ./src/proto/* && rm -rf dist && rm -rf node_modules", "update_api": "git submodule update --init --recursive && git submodule update --remote --recursive --rebase && git submodule foreach --recursive git reset --hard origin/main", + "init_api": "git submodule update --init --recursive", "protoc": "./scripts/protoc.sh", "lint": "node ./node_modules/eslint/bin/eslint src/ --ext .ts", "lint-fix": "npx eslint --ext .ts --fix src/", @@ -71,7 +72,7 @@ "test": "jest --runInBand --coverage --silent --detectOpenHandles", "prettier-check": "npx prettier --check .", "all": "npm run clean && npm run build && npm run prettier-check && npm run lint && npm run test", - "prepare": "npm run update_api && npm run protoc && npm run tsc", + "prepare": "npm run init_api && npm run protoc && npm run tsc", "prepublishOnly": "npm test && npm run lint", "prettify": "npx prettier --write .", "preversion": "npm run lint" diff --git a/src/tigris.ts b/src/tigris.ts index b343e0a..13c206c 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -268,12 +268,13 @@ export class Tigris { * * @param dbName - The name of the database to create the collections in. */ - public async registerSchemas(schemaPath: string, dbName: string) { + public async registerSchemas(schemaPath: string) { if (!path.isAbsolute(schemaPath)) { schemaPath = path.join(appRootPath.toString(), schemaPath); } // create DB + const dbName = this._config.projectName; const tigrisDb = await this.createDatabaseIfNotExists(dbName); Log.event(`Created database: ${dbName}`); From 7f0ca91b57f5bd059bb190a23b5072d6cbd40623 Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Thu, 1 Dec 2022 20:19:03 -0800 Subject: [PATCH 08/15] refactor! [BREAKING] Switching to Project paradigm (#178) --- api/proto | 2 +- src/__tests__/test-service.ts | 134 +++++++++++---------------- src/__tests__/tigris.utility.spec.ts | 4 +- src/collection.ts | 10 +- src/db.ts | 16 ++-- src/session.ts | 4 +- src/tigris.ts | 8 +- src/utility.ts | 2 +- 8 files changed, 75 insertions(+), 105 deletions(-) diff --git a/api/proto b/api/proto index e28162d..b64ce44 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit e28162d35c3ffa0ce20a3dd3675bf4e34b3f2711 +Subproject commit b64ce44e8613a46c000232dcf2b9f6eeaae396d1 diff --git a/src/__tests__/test-service.ts b/src/__tests__/test-service.ts index 7eb5abd..d45856d 100644 --- a/src/__tests__/test-service.ts +++ b/src/__tests__/test-service.ts @@ -9,34 +9,30 @@ import { CollectionMetadata, CommitTransactionRequest, CommitTransactionResponse, - CreateDatabaseRequest, - CreateDatabaseResponse, + CreateProjectRequest, + CreateProjectResponse, CreateOrUpdateCollectionRequest, CreateOrUpdateCollectionResponse, - DatabaseInfo, - DatabaseMetadata, + ProjectInfo, + ProjectMetadata, DeleteRequest, DeleteResponse, DescribeCollectionRequest, DescribeCollectionResponse, - DescribeDatabaseRequest, - DescribeDatabaseResponse, + DescribeProjectRequest, + DescribeProjectResponse, DropCollectionRequest, DropCollectionResponse, - DropDatabaseRequest, - DropDatabaseResponse, - EventsRequest, - EventsResponse, + DeleteProjectRequest, + DeleteProjectResponse, FacetCount, InsertRequest, InsertResponse, ListCollectionsRequest, ListCollectionsResponse, - ListDatabasesRequest, - ListDatabasesResponse, + ListProjectsRequest, + ListProjectsResponse, Page, - PublishRequest, - PublishResponse, ReadRequest, ReadResponse, ReplaceRequest, @@ -50,9 +46,6 @@ import { SearchMetadata, SearchRequest, SearchResponse, - StreamEvent, - SubscribeRequest, - SubscribeResponse, TransactionCtx, UpdateRequest, UpdateResponse @@ -61,7 +54,7 @@ import * as google_protobuf_timestamp_pb from "google-protobuf/google/protobuf/t import {Utility} from "../utility"; export class TestTigrisService { - private static DBS: string[] = []; + private static PROJECTS: string[] = []; private static COLLECTION_MAP = new Map>(); private static txId: string; private static txOrigin: string; @@ -86,12 +79,12 @@ export class TestTigrisService { ]); static reset() { - TestTigrisService.DBS = []; + TestTigrisService.PROJECTS = []; TestTigrisService.COLLECTION_MAP = new Map>(); this.txId = ""; this.txOrigin = ""; for (let d = 1; d <= 5; d++) { - TestTigrisService.DBS.push("db" + d); + TestTigrisService.PROJECTS.push("db" + d); const collections: string[] = []; for (let c = 1; c <= 5; c++) { collections[c - 1] = "db" + d + "_coll_" + c; @@ -101,35 +94,12 @@ export class TestTigrisService { } public impl: ITigrisServer = { - // eslint-disable-next-line @typescript-eslint/no-empty-function - events(call: ServerWritableStream): void { - const event = new StreamEvent(); - event.setTxId(Utility._base64Encode(uuidv4())); - event.setCollection("books"); - event.setOp("insert"); - event.setData(TestTigrisService.BOOKS_B64_BY_ID.get("5")); - event.setLast(true); - call.write(new EventsResponse().setEvent(event)); - call.end(); - }, - // eslint-disable-next-line @typescript-eslint/no-empty-function,@typescript-eslint/no-unused-vars - publish(call: ServerUnaryCall, callback: sendUnaryData): void { - const reply: PublishResponse = new PublishResponse(); - callback(undefined, reply); - }, - // eslint-disable-next-line @typescript-eslint/no-empty-function,@typescript-eslint/no-unused-vars - subscribe(call: ServerWritableStream): void { - for (const alert of TestTigrisService.ALERTS_B64_BY_ID) { - call.write(new SubscribeResponse().setMessage(alert[1])); - } - call.end(); - }, beginTransaction( call: ServerUnaryCall, callback: sendUnaryData ): void { const reply: BeginTransactionResponse = new BeginTransactionResponse(); - if (call.request.getDb() === "test-tx") { + if (call.request.getProject() === "test-tx") { TestTigrisService.txId = uuidv4(); TestTigrisService.txOrigin = uuidv4(); reply.setTxCtx( @@ -150,13 +120,13 @@ export class TestTigrisService { reply.setStatus("committed-test"); callback(undefined, reply); }, - createDatabase( - call: ServerUnaryCall, - callback: sendUnaryData + createProject( + call: ServerUnaryCall, + callback: sendUnaryData ): void { - TestTigrisService.DBS.push(call.request.getDb()); - const reply: CreateDatabaseResponse = new CreateDatabaseResponse(); - reply.setMessage(call.request.getDb() + " created successfully"); + TestTigrisService.PROJECTS.push(call.request.getProject()); + const reply: CreateProjectResponse = new CreateProjectResponse(); + reply.setMessage(call.request.getProject() + " created successfully"); reply.setStatus("created"); callback(undefined, reply); }, @@ -177,7 +147,7 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { - if (call.request.getDb() === "test-tx") { + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); if (txIdHeader != TestTigrisService.txId || txOriginHeader != TestTigrisService.txOrigin) { @@ -204,27 +174,27 @@ export class TestTigrisService { }, /* eslint-enable @typescript-eslint/no-empty-function */ - describeDatabase( - call: ServerUnaryCall, - callback: sendUnaryData + describeProject( + call: ServerUnaryCall, + callback: sendUnaryData ): void { - const result: DescribeDatabaseResponse = new DescribeDatabaseResponse(); + const result: DescribeProjectResponse = new DescribeProjectResponse(); const collectionsDescription: CollectionDescription[] = []; for ( let index = 0; - index < TestTigrisService.COLLECTION_MAP.get(call.request.getDb()).length; + index < TestTigrisService.COLLECTION_MAP.get(call.request.getProject()).length; index++ ) { collectionsDescription.push( new CollectionDescription() - .setCollection(TestTigrisService.COLLECTION_MAP.get(call.request.getDb())[index]) + .setCollection(TestTigrisService.COLLECTION_MAP.get(call.request.getProject())[index]) .setMetadata(new CollectionMetadata()) .setSchema("schema" + index) ); } result - .setDb(call.request.getDb()) - .setMetadata(new DatabaseMetadata()) + .setProject(call.request.getProject()) + .setMetadata(new ProjectMetadata()) .setCollectionsList(collectionsDescription); callback(undefined, result); }, @@ -233,24 +203,24 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { - const newCollections = TestTigrisService.COLLECTION_MAP.get(call.request.getDb()).filter( + const newCollections = TestTigrisService.COLLECTION_MAP.get(call.request.getProject()).filter( (coll) => coll !== call.request.getCollection() ); - TestTigrisService.COLLECTION_MAP.set(call.request.getDb(), newCollections); + TestTigrisService.COLLECTION_MAP.set(call.request.getProject(), newCollections); const reply: DropCollectionResponse = new DropCollectionResponse(); reply.setMessage(call.request.getCollection() + " dropped successfully"); reply.setStatus("dropped"); callback(undefined, reply); }, - dropDatabase( - call: ServerUnaryCall, - callback: sendUnaryData + deleteProject( + call: ServerUnaryCall, + callback: sendUnaryData ): void { - TestTigrisService.DBS = TestTigrisService.DBS.filter( - (database) => database !== call.request.getDb() + TestTigrisService.PROJECTS = TestTigrisService.PROJECTS.filter( + (database) => database !== call.request.getProject() ); - const reply: DropDatabaseResponse = new DropDatabaseResponse(); - reply.setMessage(call.request.getDb() + " dropped successfully"); + const reply: DeleteProjectResponse = new DeleteProjectResponse(); + reply.setMessage(call.request.getProject() + " dropped successfully"); reply.setStatus("dropped"); callback(undefined, reply); }, @@ -258,7 +228,7 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { - if (call.request.getDb() === "test-tx") { + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); if (txIdHeader != TestTigrisService.txId || txOriginHeader != TestTigrisService.txOrigin) { @@ -299,36 +269,36 @@ export class TestTigrisService { const collectionInfos: CollectionInfo[] = []; for ( let index = 0; - index < TestTigrisService.COLLECTION_MAP.get(call.request.getDb()).length; + index < TestTigrisService.COLLECTION_MAP.get(call.request.getProject()).length; index++ ) { collectionInfos.push( new CollectionInfo() - .setCollection(TestTigrisService.COLLECTION_MAP.get(call.request.getDb())[index]) + .setCollection(TestTigrisService.COLLECTION_MAP.get(call.request.getProject())[index]) .setMetadata(new CollectionMetadata()) ); } reply.setCollectionsList(collectionInfos); callback(undefined, reply); }, - listDatabases( - call: ServerUnaryCall, - callback: sendUnaryData + listProjects( + call: ServerUnaryCall, + callback: sendUnaryData ): void { - const reply: ListDatabasesResponse = new ListDatabasesResponse(); - const databaseInfos: DatabaseInfo[] = []; - for (let index = 0; index < TestTigrisService.DBS.length; index++) { + const reply: ListProjectsResponse = new ListProjectsResponse(); + const databaseInfos: ProjectInfo[] = []; + for (let index = 0; index < TestTigrisService.PROJECTS.length; index++) { databaseInfos.push( - new DatabaseInfo().setDb(TestTigrisService.DBS[index]).setMetadata(new DatabaseMetadata()) + new ProjectInfo().setProject(TestTigrisService.PROJECTS[index]).setMetadata(new ProjectMetadata()) ); } - reply.setDatabasesList(databaseInfos); + reply.setProjectsList(databaseInfos); callback(undefined, reply); }, // eslint-disable-next-line @typescript-eslint/no-empty-function read(call: ServerWritableStream): void { - if (call.request.getDb() === "test-tx") { + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); if (txIdHeader != TestTigrisService.txId || txOriginHeader != TestTigrisService.txOrigin) { @@ -445,7 +415,7 @@ export class TestTigrisService { // eslint-disable-next-line @typescript-eslint/no-unused-vars callback: sendUnaryData ): void { - if (call.request.getDb() === "test-tx") { + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); if (txIdHeader != TestTigrisService.txId || txOriginHeader != TestTigrisService.txOrigin) { @@ -488,7 +458,7 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { - if (call.request.getDb() === "test-tx") { + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); if (txIdHeader != TestTigrisService.txId || txOriginHeader != TestTigrisService.txOrigin) { diff --git a/src/__tests__/tigris.utility.spec.ts b/src/__tests__/tigris.utility.spec.ts index c92355d..7c73b0c 100644 --- a/src/__tests__/tigris.utility.spec.ts +++ b/src/__tests__/tigris.utility.spec.ts @@ -98,10 +98,10 @@ describe("utility tests", () => { const dbName = "my_test_db"; const collectionName = "my_test_collection"; - it("populates dbName and collection name", () => { + it("populates projectName and collection name", () => { const emptyRequest = {q: ""}; const generated = Utility.createProtoSearchRequest(dbName, collectionName, emptyRequest); - expect(generated.getDb()).toBe(dbName); + expect(generated.getProject()).toBe(dbName); expect(generated.getCollection()).toBe(collectionName); }); diff --git a/src/collection.ts b/src/collection.ts index 4d117b1..c69e071 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -72,7 +72,7 @@ export abstract class ReadOnlyCollection impleme } const readRequest = new ProtoReadRequest() - .setDb(this.db) + .setProject(this.db) .setCollection(this.collectionName) .setFilter(Utility.stringToUint8Array(Utility.filterToString(filter))); @@ -228,7 +228,7 @@ export class Collection extends ReadOnlyCollecti } const protoRequest = new ProtoInsertRequest() - .setDb(this.db) + .setProject(this.db) .setCollection(this.collectionName) .setDocumentsList(docsArray); @@ -280,7 +280,7 @@ export class Collection extends ReadOnlyCollecti docsArray.push(new TextEncoder().encode(Utility.objToJsonString(doc))); } const protoRequest = new ProtoReplaceRequest() - .setDb(this.db) + .setProject(this.db) .setCollection(this.collectionName) .setDocumentsList(docsArray); @@ -332,7 +332,7 @@ export class Collection extends ReadOnlyCollecti reject(new Error("No filter specified")); } const deleteRequest = new ProtoDeleteRequest() - .setDb(this.db) + .setProject(this.db) .setCollection(this.collectionName) .setFilter(Utility.stringToUint8Array(Utility.filterToString(filter))); @@ -391,7 +391,7 @@ export class Collection extends ReadOnlyCollecti ): Promise { return new Promise((resolve, reject) => { const updateRequest = new ProtoUpdateRequest() - .setDb(this.db) + .setProject(this.db) .setCollection(this.collectionName) .setFilter(Utility.stringToUint8Array(Utility.filterToString(filter))) .setFields(Utility.stringToUint8Array(Utility.updateFieldsString(fields))); diff --git a/src/db.ts b/src/db.ts index a8f69e9..f16df08 100644 --- a/src/db.ts +++ b/src/db.ts @@ -18,7 +18,7 @@ import { BeginTransactionResponse, CollectionOptions as ProtoCollectionOptions, CreateOrUpdateCollectionRequest as ProtoCreateOrUpdateCollectionRequest, - DescribeDatabaseRequest as ProtoDescribeDatabaseRequest, + DescribeProjectRequest as ProtoDescribeProjectRequest, DropCollectionRequest as ProtoDropCollectionRequest, ListCollectionsRequest as ProtoListCollectionsRequest, } from "./proto/server/v1/api_pb"; @@ -67,7 +67,7 @@ export class DB { const rawJSONSchema: string = Utility._toJSONSchema(name, schema); Log.debug(rawJSONSchema); const createOrUpdateCollectionRequest = new ProtoCreateOrUpdateCollectionRequest() - .setDb(this._db) + .setProject(this._db) .setCollection(name) .setOnlyCreate(false) .setSchema(Utility.stringToUint8Array(rawJSONSchema)); @@ -88,7 +88,7 @@ export class DB { public listCollections(options?: CollectionOptions): Promise> { return new Promise>((resolve, reject) => { - const request = new ProtoListCollectionsRequest().setDb(this.db); + const request = new ProtoListCollectionsRequest().setProject(this.db); if (typeof options !== "undefined") { return request.setOptions(new ProtoCollectionOptions()); } @@ -111,7 +111,7 @@ export class DB { public dropCollection(collectionName: string): Promise { return new Promise((resolve, reject) => { this.grpcClient.dropCollection( - new ProtoDropCollectionRequest().setDb(this.db).setCollection(collectionName), + new ProtoDropCollectionRequest().setProject(this.db).setCollection(collectionName), (error, response) => { if (error) { reject(error); @@ -138,8 +138,8 @@ export class DB { public describe(): Promise { return new Promise((resolve, reject) => { - this.grpcClient.describeDatabase( - new ProtoDescribeDatabaseRequest().setDb(this.db), + this.grpcClient.describeProject( + new ProtoDescribeProjectRequest().setProject(this.db), (error, response) => { if (error) { reject(error); @@ -156,7 +156,7 @@ export class DB { } resolve( new DatabaseDescription( - response.getDb(), + response.getProject(), new DatabaseMetadata(), collectionsDescription ) @@ -198,7 +198,7 @@ export class DB { // eslint-disable-next-line @typescript-eslint/no-unused-vars public beginTransaction(_options?: TransactionOptions): Promise { return new Promise((resolve, reject) => { - const beginTxRequest = new ProtoBeginTransactionRequest().setDb(this._db); + const beginTxRequest = new ProtoBeginTransactionRequest().setProject(this._db); const cookie: Metadata = new Metadata(); const call = this.grpcClient.makeUnaryRequest( BeginTransactionMethodName, diff --git a/src/session.ts b/src/session.ts index d007456..7965034 100644 --- a/src/session.ts +++ b/src/session.ts @@ -42,7 +42,7 @@ export class Session { public commit(): Promise { return new Promise((resolve, reject) => { - const request = new ProtoCommitTransactionRequest().setDb(this.db); + const request = new ProtoCommitTransactionRequest().setProject(this.db); this.grpcClient.commitTransaction(request, Utility.txToMetadata(this), (error, response) => { if (error) { reject(error); @@ -55,7 +55,7 @@ export class Session { public rollback(): Promise { return new Promise((resolve, reject) => { - const request = new ProtoRollbackTransactionRequest().setDb(this.db); + const request = new ProtoRollbackTransactionRequest().setProject(this.db); this.grpcClient.rollbackTransaction( request, Utility.txToMetadata(this), diff --git a/src/tigris.ts b/src/tigris.ts index 13c206c..b9a282b 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -4,8 +4,8 @@ import { HealthAPIClient } from "./proto/server/v1/health_grpc_pb"; import * as grpc from "@grpc/grpc-js"; import { ChannelCredentials, Metadata, status } from "@grpc/grpc-js"; import { - CreateDatabaseRequest as ProtoCreateDatabaseRequest, - DatabaseOptions as ProtoDatabaseOptions, + CreateProjectRequest as ProtoCreateProjectRequest, + ProjectOptions as ProtoProjectOptions, } from "./proto/server/v1/api_pb"; import { GetInfoRequest as ProtoGetInfoRequest } from "./proto/server/v1/observability_pb"; import { HealthCheckInput as ProtoHealthCheckInput } from "./proto/server/v1/health_pb"; @@ -297,8 +297,8 @@ export class Tigris { // eslint-disable-next-line @typescript-eslint/no-unused-vars private createDatabaseIfNotExists(db: string, _options?: DatabaseOptions): Promise { return new Promise((resolve, reject) => { - this.grpcClient.createDatabase( - new ProtoCreateDatabaseRequest().setDb(db).setOptions(new ProtoDatabaseOptions()), + this.grpcClient.createProject( + new ProtoCreateProjectRequest().setProject(db).setOptions(new ProtoProjectOptions()), // eslint-disable-next-line @typescript-eslint/no-unused-vars (error, _response) => { if (error && error.code != status.ALREADY_EXISTS) { diff --git a/src/utility.ts b/src/utility.ts index bdf3d2f..f8c521a 100644 --- a/src/utility.ts +++ b/src/utility.ts @@ -505,7 +505,7 @@ export const Utility = { options?: SearchRequestOptions ): ProtoSearchRequest { const searchRequest = new ProtoSearchRequest() - .setDb(dbName) + .setProject(dbName) .setCollection(collectionName) .setQ(request.q ?? MATCH_ALL_QUERY_STRING); From cbad6083885d80de5feb1377cdfb207888e32f8c Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Sun, 4 Dec 2022 15:59:05 -0800 Subject: [PATCH 09/15] refactor! [BREAKING] Switching to Project paradigm (#180) --- api/proto | 2 +- src/__tests__/test-service.ts | 15 ++++++--------- src/__tests__/tigris.rpc.spec.ts | 1 - src/db.ts | 14 ++++---------- src/tigris.ts | 7 ++----- src/types.ts | 12 +----------- 6 files changed, 14 insertions(+), 37 deletions(-) diff --git a/api/proto b/api/proto index b64ce44..58c9133 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit b64ce44e8613a46c000232dcf2b9f6eeaae396d1 +Subproject commit 58c9133be42833131246b3d47f1c20cbe131c993 diff --git a/src/__tests__/test-service.ts b/src/__tests__/test-service.ts index d45856d..3e04957 100644 --- a/src/__tests__/test-service.ts +++ b/src/__tests__/test-service.ts @@ -19,8 +19,6 @@ import { DeleteResponse, DescribeCollectionRequest, DescribeCollectionResponse, - DescribeProjectRequest, - DescribeProjectResponse, DropCollectionRequest, DropCollectionResponse, DeleteProjectRequest, @@ -48,7 +46,7 @@ import { SearchResponse, TransactionCtx, UpdateRequest, - UpdateResponse + UpdateResponse, DescribeDatabaseRequest, DescribeDatabaseResponse } from "../proto/server/v1/api_pb"; import * as google_protobuf_timestamp_pb from "google-protobuf/google/protobuf/timestamp_pb"; import {Utility} from "../utility"; @@ -172,13 +170,13 @@ export class TestTigrisService { _callback: sendUnaryData ): void { }, - /* eslint-enable @typescript-eslint/no-empty-function */ - describeProject( - call: ServerUnaryCall, - callback: sendUnaryData + /* eslint-enable @typescript-eslint/no-empty-function */ + describeDatabase( + call: ServerUnaryCall, + callback: sendUnaryData ): void { - const result: DescribeProjectResponse = new DescribeProjectResponse(); + const result: DescribeDatabaseResponse = new DescribeDatabaseResponse(); const collectionsDescription: CollectionDescription[] = []; for ( let index = 0; @@ -193,7 +191,6 @@ export class TestTigrisService { ); } result - .setProject(call.request.getProject()) .setMetadata(new ProjectMetadata()) .setCollectionsList(collectionsDescription); callback(undefined, result); diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index 2a71daa..bf66031 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -96,7 +96,6 @@ describe("rpc tests", () => { const databaseDescriptionPromise = db1.describe(); databaseDescriptionPromise.then(value => { - expect(value.db).toBe("db3"); expect(value.collectionsDescription.length).toBe(5); expect(value.collectionsDescription[0].collection).toBe("db3_coll_1"); expect(value.collectionsDescription[1].collection).toBe("db3_coll_2"); diff --git a/src/db.ts b/src/db.ts index f16df08..e62bb3e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -18,7 +18,7 @@ import { BeginTransactionResponse, CollectionOptions as ProtoCollectionOptions, CreateOrUpdateCollectionRequest as ProtoCreateOrUpdateCollectionRequest, - DescribeProjectRequest as ProtoDescribeProjectRequest, + DescribeDatabaseRequest as ProtoDescribeDatabaseRequest, DropCollectionRequest as ProtoDropCollectionRequest, ListCollectionsRequest as ProtoListCollectionsRequest, } from "./proto/server/v1/api_pb"; @@ -138,8 +138,8 @@ export class DB { public describe(): Promise { return new Promise((resolve, reject) => { - this.grpcClient.describeProject( - new ProtoDescribeProjectRequest().setProject(this.db), + this.grpcClient.describeDatabase( + new ProtoDescribeDatabaseRequest().setProject(this.db), (error, response) => { if (error) { reject(error); @@ -154,13 +154,7 @@ export class DB { ) ); } - resolve( - new DatabaseDescription( - response.getProject(), - new DatabaseMetadata(), - collectionsDescription - ) - ); + resolve(new DatabaseDescription(new DatabaseMetadata(), collectionsDescription)); } } ); diff --git a/src/tigris.ts b/src/tigris.ts index b9a282b..8fbd4c3 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -3,10 +3,7 @@ import { ObservabilityClient } from "./proto/server/v1/observability_grpc_pb"; import { HealthAPIClient } from "./proto/server/v1/health_grpc_pb"; import * as grpc from "@grpc/grpc-js"; import { ChannelCredentials, Metadata, status } from "@grpc/grpc-js"; -import { - CreateProjectRequest as ProtoCreateProjectRequest, - ProjectOptions as ProtoProjectOptions, -} from "./proto/server/v1/api_pb"; +import { CreateProjectRequest as ProtoCreateProjectRequest } from "./proto/server/v1/api_pb"; import { GetInfoRequest as ProtoGetInfoRequest } from "./proto/server/v1/observability_pb"; import { HealthCheckInput as ProtoHealthCheckInput } from "./proto/server/v1/health_pb"; @@ -298,7 +295,7 @@ export class Tigris { private createDatabaseIfNotExists(db: string, _options?: DatabaseOptions): Promise { return new Promise((resolve, reject) => { this.grpcClient.createProject( - new ProtoCreateProjectRequest().setProject(db).setOptions(new ProtoProjectOptions()), + new ProtoCreateProjectRequest().setProject(db), // eslint-disable-next-line @typescript-eslint/no-unused-vars (error, _response) => { if (error && error.code != status.ALREADY_EXISTS) { diff --git a/src/types.ts b/src/types.ts index 9a98c86..a899c3c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -81,24 +81,14 @@ export class DropCollectionResponse { } export class DatabaseDescription { - private readonly _db: string; private readonly _metadata: DatabaseMetadata; private readonly _collectionsDescription: Array; - constructor( - db: string, - metadata: DatabaseMetadata, - collectionsDescription: Array - ) { - this._db = db; + constructor(metadata: DatabaseMetadata, collectionsDescription: Array) { this._metadata = metadata; this._collectionsDescription = collectionsDescription; } - get db(): string { - return this._db; - } - get metadata(): DatabaseMetadata { return this._metadata; } From 220c6c46945fbc9474080fadbb24060ff1da5135 Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Tue, 6 Dec 2022 18:16:06 -0800 Subject: [PATCH 10/15] refactor: DatabaseMetadata and ProjectMetadata (#181) --- api/proto | 2 +- src/__tests__/test-service.ts | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/api/proto b/api/proto index 58c9133..d9d2608 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit 58c9133be42833131246b3d47f1c20cbe131c993 +Subproject commit d9d2608f582edfa9dc95f89d169b5c0d26e68360 diff --git a/src/__tests__/test-service.ts b/src/__tests__/test-service.ts index 3e04957..0f40e35 100644 --- a/src/__tests__/test-service.ts +++ b/src/__tests__/test-service.ts @@ -14,7 +14,7 @@ import { CreateOrUpdateCollectionRequest, CreateOrUpdateCollectionResponse, ProjectInfo, - ProjectMetadata, + DatabaseMetadata, DeleteRequest, DeleteResponse, DescribeCollectionRequest, @@ -46,7 +46,11 @@ import { SearchResponse, TransactionCtx, UpdateRequest, - UpdateResponse, DescribeDatabaseRequest, DescribeDatabaseResponse + UpdateResponse, + DescribeDatabaseRequest, + DescribeDatabaseResponse, + CreateBranchRequest, + CreateBranchResponse, DeleteBranchRequest, DeleteBranchResponse } from "../proto/server/v1/api_pb"; import * as google_protobuf_timestamp_pb from "google-protobuf/google/protobuf/timestamp_pb"; import {Utility} from "../utility"; @@ -92,6 +96,18 @@ export class TestTigrisService { } public impl: ITigrisServer = { + createBranch( + call: ServerUnaryCall, + callback: sendUnaryData + ): void { + // TODO implement + }, + deleteBranch( + call: ServerUnaryCall, + callback: sendUnaryData + ): void { + // TODO implement + }, beginTransaction( call: ServerUnaryCall, callback: sendUnaryData @@ -191,7 +207,7 @@ export class TestTigrisService { ); } result - .setMetadata(new ProjectMetadata()) + .setMetadata(new DatabaseMetadata()) .setCollectionsList(collectionsDescription); callback(undefined, result); }, @@ -286,7 +302,7 @@ export class TestTigrisService { const databaseInfos: ProjectInfo[] = []; for (let index = 0; index < TestTigrisService.PROJECTS.length; index++) { databaseInfos.push( - new ProjectInfo().setProject(TestTigrisService.PROJECTS[index]).setMetadata(new ProjectMetadata()) + new ProjectInfo().setProject(TestTigrisService.PROJECTS[index]).setMetadata(new DatabaseMetadata()) ); } From bc412914fa1d95b0bcb81d144c8e8b97a377417d Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Mon, 12 Dec 2022 16:05:36 -0800 Subject: [PATCH 11/15] fix: Export Schema related types (#182) --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index b8e42bb..9f688bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,3 +2,4 @@ export * from "./collection"; export * from "./db"; export * from "./session"; export * from "./tigris"; +export * from "./types"; From 05808cb60487d7a459b84f1af1e67834bf923c2c Mon Sep 17 00:00:00 2001 From: Adil Ansari Date: Thu, 15 Dec 2022 10:23:00 -0800 Subject: [PATCH 12/15] feat: schema generation using decorators (#183) * Decorators proposal with schema examples * Maxlen option for byte string --- .eslintrc.json | 2 +- jest.config.json | 9 +- package-lock.json | 11 ++ package.json | 1 + .../data/decoratedModels/matrices.ts | 85 +++++++++ src/__tests__/data/decoratedModels/movies.ts | 106 ++++++++++++ src/__tests__/data/decoratedModels/orders.ts | 101 +++++++++++ src/__tests__/data/decoratedModels/users.ts | 88 ++++++++++ .../schema/decorator-processor.spec.ts | 40 +++++ src/__tests__/tigris.rpc.spec.ts | 16 +- src/db.ts | 68 +++++++- .../metadata/collection-metadata.ts | 5 + .../metadata/decorator-meta-storage.ts | 36 ++++ src/decorators/metadata/field-metadata.ts | 11 ++ .../metadata/primary-key-metadata.ts | 9 + .../options/embedded-field-options.ts | 17 ++ src/decorators/tigris-collection.ts | 15 ++ src/decorators/tigris-field.ts | 161 ++++++++++++++++++ src/decorators/tigris-primary-key.ts | 82 +++++++++ src/error.ts | 45 +++++ src/globals.ts | 9 + src/index.ts | 6 + src/schema/decorated-schema-processor.ts | 104 +++++++++++ src/types.ts | 8 +- tsconfig.json | 1 + 25 files changed, 1023 insertions(+), 13 deletions(-) create mode 100644 src/__tests__/data/decoratedModels/matrices.ts create mode 100644 src/__tests__/data/decoratedModels/movies.ts create mode 100644 src/__tests__/data/decoratedModels/orders.ts create mode 100644 src/__tests__/data/decoratedModels/users.ts create mode 100644 src/__tests__/schema/decorator-processor.spec.ts create mode 100644 src/decorators/metadata/collection-metadata.ts create mode 100644 src/decorators/metadata/decorator-meta-storage.ts create mode 100644 src/decorators/metadata/field-metadata.ts create mode 100644 src/decorators/metadata/primary-key-metadata.ts create mode 100644 src/decorators/options/embedded-field-options.ts create mode 100644 src/decorators/tigris-collection.ts create mode 100644 src/decorators/tigris-field.ts create mode 100644 src/decorators/tigris-primary-key.ts create mode 100644 src/globals.ts create mode 100644 src/schema/decorated-schema-processor.ts diff --git a/.eslintrc.json b/.eslintrc.json index 8e71b4b..30db966 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -21,7 +21,7 @@ "@typescript-eslint/ban-types": [ "error", { - "types": { "BigInt": false }, + "types": { "BigInt": false, "Function": false, "Object": false }, "extendDefaults": true } ], diff --git a/jest.config.json b/jest.config.json index 969a33f..1deba5a 100644 --- a/jest.config.json +++ b/jest.config.json @@ -9,5 +9,12 @@ "collectCoverage": true, "coverageDirectory": "coverage", "coverageProvider": "v8", - "collectCoverageFrom": ["src/*.ts", "src/consumables/*.ts", "src/utils/*.ts", "src/search/*.ts"] + "collectCoverageFrom": [ + "src/*.ts", + "src/consumables/*.ts", + "src/decorators/**/*.ts", + "src/schema/**/*.ts", + "src/search/**/*.ts", + "src/utils/**/*.ts" + ] } diff --git a/package-lock.json b/package-lock.json index f031d00..560d10c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "dotenv": "^16.0.3", "google-protobuf": "^3.21.0", "json-bigint": "^1.0.0", + "reflect-metadata": "^0.1.13", "typescript": "^4.7.2" }, "devDependencies": { @@ -10070,6 +10071,11 @@ "esprima": "~4.0.0" } }, + "node_modules/reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" + }, "node_modules/regexp-tree": { "version": "0.1.24", "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.24.tgz", @@ -18917,6 +18923,11 @@ "esprima": "~4.0.0" } }, + "reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" + }, "regexp-tree": { "version": "0.1.24", "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.24.tgz", diff --git a/package.json b/package.json index c4b3f60..82f00cb 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "dotenv": "^16.0.3", "google-protobuf": "^3.21.0", "json-bigint": "^1.0.0", + "reflect-metadata": "^0.1.13", "typescript": "^4.7.2" } } diff --git a/src/__tests__/data/decoratedModels/matrices.ts b/src/__tests__/data/decoratedModels/matrices.ts new file mode 100644 index 0000000..e424caa --- /dev/null +++ b/src/__tests__/data/decoratedModels/matrices.ts @@ -0,0 +1,85 @@ +import { Field } from "../../../decorators/tigris-field"; +import { TigrisCollectionType, TigrisDataTypes, TigrisSchema } from "../../../types"; +import { TigrisCollection } from "../../../decorators/tigris-collection"; +import { PrimaryKey } from "../../../decorators/tigris-primary-key"; + +/****************************************************************************** + * `Matrix` class demonstrates a Tigris collection schema generated using + * decorators. Type of collection fields is inferred using Reflection APIs. This + * particular schema example: + * - has a nested Array (Array of Arrays) + * - infers the type of collection fields automatically using Reflection APIs + *****************************************************************************/ +export class CellValue { + @Field() + length: number + + @Field() + type: string; +} + +export class Cell { + @Field() + x: number; + + @Field() + y: number; + + @Field() + value: CellValue; +} + +@TigrisCollection("matrices") +export class Matrix implements TigrisCollectionType { + @PrimaryKey({order: 1}) + id: string; + + @Field({elements: Cell, depth: 3}) + cells: Array>>; +} +/********************************** END **************************************/ + +/** + * `TigrisSchema` representation of the collection class above. + * + * NOTE: This is only an illustration; you don't have to write this definition, + * it will be auto generated. + */ +export const ExpectedSchema: TigrisSchema = { + id: { + type: TigrisDataTypes.STRING, + primary_key: { + order: 1, + autoGenerate: false + }, + }, + cells: { + type: TigrisDataTypes.ARRAY, + items: { + type: TigrisDataTypes.ARRAY, + items: { + type: TigrisDataTypes.ARRAY, + items: { + type: { + x: { + type: TigrisDataTypes.NUMBER, + }, + y: { + type: TigrisDataTypes.NUMBER, + }, + value: { + type: { + length: { + type: TigrisDataTypes.NUMBER + }, + type: { + type: TigrisDataTypes.STRING + } + }, + }, + } + } + } + } + } +} diff --git a/src/__tests__/data/decoratedModels/movies.ts b/src/__tests__/data/decoratedModels/movies.ts new file mode 100644 index 0000000..db074ca --- /dev/null +++ b/src/__tests__/data/decoratedModels/movies.ts @@ -0,0 +1,106 @@ +import { TigrisCollection } from "../../../decorators/tigris-collection"; +import { PrimaryKey } from "../../../decorators/tigris-primary-key"; +import { TigrisDataTypes, TigrisSchema } from "../../../types"; +import { Field } from "../../../decorators/tigris-field"; + +/****************************************************************************** + * `Movie` class demonstrates a Tigris collection schema generated using + * decorators. This particular schema example: + * - has an Array of another class as embedded Object + * - has an Array of primitive types + * - has an Object of type `Studio` + * - does not use reflection, all the collection fields are explicitly typed + *****************************************************************************/ + +export class Studio { + @Field(TigrisDataTypes.STRING) + name: string; + + @Field(TigrisDataTypes.STRING) + city: string; +} + +export class Actor { + @Field(TigrisDataTypes.STRING, {maxLength: 64}) + firstName: string; + + @Field(TigrisDataTypes.STRING, {maxLength: 64}) + lastName: string; +} + +@TigrisCollection("movies") +export class Movie{ + + @PrimaryKey(TigrisDataTypes.STRING, {autoGenerate: true, order: 1}) + movieId: string; + + @Field(TigrisDataTypes.STRING) + title: string; + + @Field(TigrisDataTypes.INT32) + year: number; + + @Field(TigrisDataTypes.ARRAY, {elements: Actor}) + actors: Array; + + @Field(TigrisDataTypes.ARRAY, {elements: TigrisDataTypes.STRING}) + genres: Array; + + @Field(TigrisDataTypes.OBJECT, {elements: Studio}) + productionHouse: Studio; +} + +/********************************** END **************************************/ + +/** + * `TigrisSchema` representation of the collection class above. + * + * NOTE: This is only an illustration; you don't have to write this definition, + * it will be auto generated. + */ +export const ExpectedSchema: TigrisSchema = { + movieId: { + type: TigrisDataTypes.STRING, + primary_key: { + order: 1, + autoGenerate: true + } + }, + title: { + type: TigrisDataTypes.STRING + }, + year: { + type: TigrisDataTypes.INT32 + }, + actors: { + type: TigrisDataTypes.ARRAY, + items: { + type: { + firstName: { + type: TigrisDataTypes.STRING, + maxLength: 64 + }, + lastName: { + type: TigrisDataTypes.STRING, + maxLength: 64 + } + } + } + }, + genres: { + type: TigrisDataTypes.ARRAY, + items: { + type: TigrisDataTypes.STRING + } + }, + productionHouse: { + type: { + name: { + type: TigrisDataTypes.STRING + }, + city: { + type: TigrisDataTypes.STRING + } + } + } +} diff --git a/src/__tests__/data/decoratedModels/orders.ts b/src/__tests__/data/decoratedModels/orders.ts new file mode 100644 index 0000000..31f0728 --- /dev/null +++ b/src/__tests__/data/decoratedModels/orders.ts @@ -0,0 +1,101 @@ +import { TigrisCollection } from "../../../decorators/tigris-collection"; +import { PrimaryKey } from "../../../decorators/tigris-primary-key"; +import { TigrisDataTypes, TigrisSchema } from "../../../types"; +import { Field } from "../../../decorators/tigris-field"; + +/****************************************************************************** + * `Order` class demonstrates a Tigris collection schema generated using + * decorators. Type of collection fields is inferred using Reflection APIs. This + * particular schema example: + * - has multiple primary keys + * - has embedded objects + * - has an Array of embedded objects + * - and infers the type of collection fields automatically using Reflection APIs + *****************************************************************************/ +export class Brand { + @Field() + name: string; + + @Field({elements: TigrisDataTypes.STRING}) + tags: Set; +} + +export class Product { + @Field() + name: string; + + @Field() + brand: Brand; + + @Field() + upc: bigint; + + @Field() + price: number; +} + +@TigrisCollection("orders") +export class Order { + @PrimaryKey(TigrisDataTypes.UUID,{order: 1, autoGenerate: true}) + orderId: string; + + @PrimaryKey({order: 2}) + customerId: string; + + @Field({elements: Product}) + products: Array +} + +/********************************** END **************************************/ + +/** + * `TigrisSchema` representation of the collection class above. + * + * NOTE: This is only an illustration; you don't have to write this definition, + * it will be auto generated. + */ +export const ExpectedSchema: TigrisSchema = { + orderId: { + type: TigrisDataTypes.UUID, + primary_key: { + order:1, + autoGenerate: true + } + }, + customerId: { + type: TigrisDataTypes.STRING, + primary_key: { + order: 2, + autoGenerate: false + } + }, + products: { + type: TigrisDataTypes.ARRAY, + items: { + type: { + name: { + type: TigrisDataTypes.STRING + }, + brand: { + type: { + name: { + type: TigrisDataTypes.STRING + }, + tags: { + type: TigrisDataTypes.ARRAY, + items: { + type: TigrisDataTypes.STRING + } + } + } + }, + upc: { + type: TigrisDataTypes.NUMBER_BIGINT + }, + price: { + type: TigrisDataTypes.NUMBER + } + } + } + } +} diff --git a/src/__tests__/data/decoratedModels/users.ts b/src/__tests__/data/decoratedModels/users.ts new file mode 100644 index 0000000..b1f7d81 --- /dev/null +++ b/src/__tests__/data/decoratedModels/users.ts @@ -0,0 +1,88 @@ +import { TigrisCollectionType, TigrisDataTypes, TigrisSchema } from "../../../types"; +import { PrimaryKey } from "../../../decorators/tigris-primary-key"; +import { Field } from "../../../decorators/tigris-field"; +import { TigrisCollection } from "../../../decorators/tigris-collection"; + +/****************************************************************************** + * `User` class demonstrates a Tigris collection schema generated using + * decorators. Type of collection fields is inferred using Reflection APIs. This + * particular schema example: + * - has an Array of embedded objects + * - has an Array of primitive types + * - infers the type of collection fields automatically using Reflection APIs + *****************************************************************************/ +export class Identity { + @Field({maxLength: 128}) + connection?: string; + + @Field() + isSocial: boolean; + + @Field({elements: TigrisDataTypes.NUMBER}) + provider: Array; + + @Field() + linkedAccounts: number; +} + +@TigrisCollection("users") +export class User implements TigrisCollectionType { + @PrimaryKey({order: 1}) + id: number; + + @Field(TigrisDataTypes.DATE_TIME) + created: string; + + @Field({elements: Identity}) + identities: Array; + + @Field() + name: string; +} + +/********************************** END **************************************/ + +/** + * `TigrisSchema` representation of the collection class above. + * + * NOTE: This is only an illustration; you don't have to write this definition, + * it will be auto generated. + */ +export const ExpectedSchema: TigrisSchema = { + id: { + type: TigrisDataTypes.NUMBER, + primary_key: { + order: 1, + autoGenerate: false + }, + }, + created: { + type: TigrisDataTypes.DATE_TIME + }, + identities: { + type: TigrisDataTypes.ARRAY, + items: { + type: { + connection: { + type: TigrisDataTypes.STRING, + maxLength: 128 + }, + isSocial: { + type: TigrisDataTypes.BOOLEAN + }, + provider: { + type: TigrisDataTypes.ARRAY, + items: { + type: TigrisDataTypes.NUMBER + } + }, + linkedAccounts: { + type: TigrisDataTypes.NUMBER + } + } + } + }, + name: { + type: TigrisDataTypes.STRING + } +}; diff --git a/src/__tests__/schema/decorator-processor.spec.ts b/src/__tests__/schema/decorator-processor.spec.ts new file mode 100644 index 0000000..35d21ce --- /dev/null +++ b/src/__tests__/schema/decorator-processor.spec.ts @@ -0,0 +1,40 @@ +import { + CollectionSchema, + DecoratedSchemaProcessor +} from "../../schema/decorated-schema-processor"; +import { TigrisCollectionType, TigrisSchema } from "../../types"; +import { ExpectedSchema as UserSchema, User } from "../data/./decoratedModels/users"; +import { ExpectedSchema as OrderSchema, Order } from "../data/./decoratedModels/orders"; +import { ExpectedSchema as MovieSchema, Movie } from "../data/./decoratedModels/movies"; +import { ExpectedSchema as MatricesSchema, Matrix } from "../data/./decoratedModels/matrices"; + +/* + * TODO: Add following tests + * + * add a constructor to class and subclasses + * readonly properties (getter/setter) + * custom constructor + * embedded definitions are empty + */ + +describe("Generate TigrisSchema from decorated classes", () => { + const processor = DecoratedSchemaProcessor.Instance; + type AnnotationTestCase = { + schemaClass: T, + expected: TigrisSchema + } + const schemaDefinitions: Array> = [ + { schemaClass: User, expected: UserSchema }, + { schemaClass: Order, expected: OrderSchema}, + { schemaClass: Movie, expected: MovieSchema}, + { schemaClass: Matrix, expected: MatricesSchema}, + ] + test.each(schemaDefinitions)( + "from %p schema", + (tc) => { + const generated: CollectionSchema = processor.process(tc.schemaClass); + expect(generated.schema).toStrictEqual(tc.expected); + // TODO: validate type compatibility + } + ); +}); diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index bf66031..f28595c 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -1,6 +1,6 @@ -import {Server, ServerCredentials} from "@grpc/grpc-js"; -import {TigrisService} from "../proto/server/v1/api_grpc_pb"; -import TestService, {TestTigrisService} from "./test-service"; +import { Server, ServerCredentials } from "@grpc/grpc-js"; +import { TigrisService } from "../proto/server/v1/api_grpc_pb"; +import TestService, { TestTigrisService } from "./test-service"; import { DeleteRequestOptions, LogicalOperator, @@ -11,12 +11,12 @@ import { UpdateFieldsOperator, UpdateRequestOptions } from "../types"; -import {Tigris} from "../tigris"; -import {Case, Collation, SearchRequest, SearchRequestOptions} from "../search/types"; -import {Utility} from "../utility"; -import {ObservabilityService} from "../proto/server/v1/observability_grpc_pb"; +import { Tigris } from "../tigris"; +import { Case, Collation, SearchRequest, SearchRequestOptions } from "../search/types"; +import { Utility } from "../utility"; +import { ObservabilityService } from "../proto/server/v1/observability_grpc_pb"; import TestObservabilityService from "./test-observability-service"; -import {capture, spy } from "ts-mockito"; +import { capture, spy } from "ts-mockito"; describe("rpc tests", () => { let server: Server; diff --git a/src/db.ts b/src/db.ts index e62bb3e..675c104 100644 --- a/src/db.ts +++ b/src/db.ts @@ -28,6 +28,7 @@ import { Utility } from "./utility"; import { Metadata, ServiceError } from "@grpc/grpc-js"; import { TigrisClientConfig } from "./tigris"; import { Log } from "./utils/logger"; +import { DecoratedSchemaProcessor } from "./schema/decorated-schema-processor"; /** * Tigris Database @@ -40,17 +41,82 @@ export class DB { private readonly _db: string; private readonly grpcClient: TigrisClient; private readonly config: TigrisClientConfig; + private readonly schemaProcessor: DecoratedSchemaProcessor; constructor(db: string, grpcClient: TigrisClient, config: TigrisClientConfig) { this._db = db; this.grpcClient = grpcClient; this.config = config; + this.schemaProcessor = DecoratedSchemaProcessor.Instance; } + /** + * Create a new collection if not exists. Else, apply schema changes, if any. + * + * @param cls - A Class representing schema fields using decorators + * + * @example + * + * ``` + * @TigrisCollection("todoItems") + * class TodoItem implements TigrisCollectionType { + * @PrimaryKey(TigrisDataTypes.INT32, { order: 1 }) + * id: number; + * + * @Field() + * text: string; + * + * @Field() + * completed: boolean; + * } + * + * await db.createOrUpdateCollection(TodoItem); + * ``` + */ + public createOrUpdateCollection( + cls: new () => TigrisCollectionType + ): Promise>; + + /** + * Create a new collection if not exists. Else, apply schema changes, if any. + * + * @param collectionName - Name of the Tigris Collection + * @param schema - Collection's data model + * + * @example + * + * ``` + * const TodoItemSchema: TigrisSchema = { + * id: { + * type: TigrisDataTypes.INT32, + * primary_key: { order: 1, autoGenerate: true } + * }, + * text: { type: TigrisDataTypes.STRING }, + * completed: { type: TigrisDataTypes.BOOLEAN } + * }; + * + * await db.createOrUpdateCollection("todoItems", TodoItemSchema); + * ``` + */ public createOrUpdateCollection( collectionName: string, schema: TigrisSchema - ): Promise> { + ): Promise>; + + public createOrUpdateCollection( + nameOrClass: string | TigrisCollectionType, + schema?: TigrisSchema + ) { + let collectionName: string; + if (typeof nameOrClass === "string") { + collectionName = nameOrClass as string; + } else { + const generatedColl = this.schemaProcessor.process( + nameOrClass as new () => TigrisCollectionType + ); + collectionName = generatedColl.name; + schema = generatedColl.schema as TigrisSchema; + } return this.createOrUpdate( collectionName, schema, diff --git a/src/decorators/metadata/collection-metadata.ts b/src/decorators/metadata/collection-metadata.ts new file mode 100644 index 0000000..7f12b17 --- /dev/null +++ b/src/decorators/metadata/collection-metadata.ts @@ -0,0 +1,5 @@ +/**@internal*/ +export interface CollectionMetadata { + readonly collectionName: string; + readonly target: Function; +} diff --git a/src/decorators/metadata/decorator-meta-storage.ts b/src/decorators/metadata/decorator-meta-storage.ts new file mode 100644 index 0000000..a178247 --- /dev/null +++ b/src/decorators/metadata/decorator-meta-storage.ts @@ -0,0 +1,36 @@ +import { CollectionMetadata } from "./collection-metadata"; +import { FieldMetadata } from "./field-metadata"; +import { PrimaryKeyMetadata } from "./primary-key-metadata"; + +/** + * Temporary storage for storing metadata processed by decorators. Classes can + * be loaded in any order, schema generation cannot start until all class metadata + * is available. + * + * @internal + */ +export class DecoratorMetaStorage { + readonly collections: Map = new Map(); + readonly fields: Array = new Array(); + readonly primaryKeys: Array = new Array(); + + filterCollectionByTarget(target: Function): CollectionMetadata { + for (const collection of this.collections.values()) { + if (collection.target === target) { + return collection; + } + } + } + + filterFieldsByTarget(target: Function): FieldMetadata[] { + return this.fields.filter(function (field) { + return field.target === target; + }); + } + + filterPKsByTarget(target: Function): PrimaryKeyMetadata[] { + return this.primaryKeys.filter(function (pk) { + return pk.target === target; + }); + } +} diff --git a/src/decorators/metadata/field-metadata.ts b/src/decorators/metadata/field-metadata.ts new file mode 100644 index 0000000..b7f460b --- /dev/null +++ b/src/decorators/metadata/field-metadata.ts @@ -0,0 +1,11 @@ +import { TigrisDataTypes, TigrisFieldOptions } from "../../types"; + +/**@internal*/ +export interface FieldMetadata { + readonly name: string; + readonly target: Function; + readonly type: TigrisDataTypes; + readonly embedType?: TigrisDataTypes | Function; + readonly arrayDepth?: number; + readonly schemaFieldOptions?: TigrisFieldOptions; +} diff --git a/src/decorators/metadata/primary-key-metadata.ts b/src/decorators/metadata/primary-key-metadata.ts new file mode 100644 index 0000000..9e58008 --- /dev/null +++ b/src/decorators/metadata/primary-key-metadata.ts @@ -0,0 +1,9 @@ +import { PrimaryKeyOptions, TigrisDataTypes } from "../../types"; + +/**@internal*/ +export interface PrimaryKeyMetadata { + readonly name: string; + readonly target: Function; + type: TigrisDataTypes; + readonly options: PrimaryKeyOptions; +} diff --git a/src/decorators/options/embedded-field-options.ts b/src/decorators/options/embedded-field-options.ts new file mode 100644 index 0000000..bfeec06 --- /dev/null +++ b/src/decorators/options/embedded-field-options.ts @@ -0,0 +1,17 @@ +import { TigrisDataTypes } from "../../types"; + +/** + * Additional type information for Arrays and Objects schema fields + */ +export type EmbeddedFieldOptions = { + elements: TigrisDataTypes | Function; + /** + * Optionally used to specify nested arrays (Array of arrays). + * + * @example + * - Array will have `depth` of 1 (need not be specified) + * - Array> will have `depth` of 2 + * - Array>> will have `depth` of 3 + */ + depth?: number; +}; diff --git a/src/decorators/tigris-collection.ts b/src/decorators/tigris-collection.ts new file mode 100644 index 0000000..ee27cbb --- /dev/null +++ b/src/decorators/tigris-collection.ts @@ -0,0 +1,15 @@ +import { getDecoratorMetaStorage } from "../globals"; + +/** + * TigrisCollection decorator is used to mark a class as a Collection's schema/data model. + * + * @param name - Name of collection + */ +export function TigrisCollection(name: string): ClassDecorator { + return function (target) { + getDecoratorMetaStorage().collections.set(name, { + collectionName: name, + target: target, + }); + }; +} diff --git a/src/decorators/tigris-field.ts b/src/decorators/tigris-field.ts new file mode 100644 index 0000000..075af9a --- /dev/null +++ b/src/decorators/tigris-field.ts @@ -0,0 +1,161 @@ +import "reflect-metadata"; +import { TigrisDataTypes, TigrisFieldOptions } from "../types"; +import { EmbeddedFieldOptions } from "./options/embedded-field-options"; +import { + CannotInferFieldTypeError, + IncompleteArrayTypeDefError, + ReflectionNotEnabled, +} from "../error"; +import { getDecoratorMetaStorage } from "../globals"; +import { FieldMetadata } from "./metadata/field-metadata"; +import { Log } from "../utils/logger"; + +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + * + * Uses `Reflection` to determine the data type of schema Field. + */ +export function Field(): PropertyDecorator; +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + * + * @param type - Schema field's data type + */ +export function Field(type: TigrisDataTypes): PropertyDecorator; +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + * + * Uses `Reflection` to determine the data type of schema Field. + * + * @param options - Optional properties of the schema field + */ +export function Field(options: TigrisFieldOptions): PropertyDecorator; +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + * + * Uses `Reflection` to determine the data type of schema Field. + * + * @param options - `EmbeddedFieldOptions` are only applicable to Array and Object types + * of schema field. + */ +export function Field(options: EmbeddedFieldOptions): PropertyDecorator; +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + * + * Uses `Reflection` to determine the data type of Field. + * + * @param type - Schema field's data type + * @param options - Optional properties of the schema field + */ +export function Field(type: TigrisDataTypes, options?: TigrisFieldOptions): PropertyDecorator; +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + * + * Uses `Reflection` to determine the data type of schema Field. + * + * @param type - Schema field's data type + * @param options - `EmbeddedFieldOptions` are only applicable to Array and Object types + * of schema field. + */ +export function Field(type: TigrisDataTypes, options?: EmbeddedFieldOptions): PropertyDecorator; + +/** + * Field decorator is used to mark a class property as Collection field. Only properties + * decorated with `@Field` will be used in Schema. + */ +export function Field( + typeOrOptions?: TigrisDataTypes | TigrisFieldOptions | EmbeddedFieldOptions, + options?: TigrisFieldOptions | EmbeddedFieldOptions +): PropertyDecorator { + return function (target, propertyName) { + propertyName = propertyName.toString(); + let propertyType: TigrisDataTypes | undefined; + let fieldOptions: TigrisFieldOptions; + let embedOptions: EmbeddedFieldOptions; + + if (typeof typeOrOptions === "string") { + propertyType = typeOrOptions; + } else if (typeof typeOrOptions === "object") { + if (isEmbeddedOption(typeOrOptions)) { + embedOptions = typeOrOptions as EmbeddedFieldOptions; + } else { + fieldOptions = typeOrOptions as TigrisFieldOptions; + } + } + + if (typeof options === "object") { + if (isEmbeddedOption(options)) { + embedOptions = options as EmbeddedFieldOptions; + } else { + fieldOptions = options as TigrisFieldOptions; + } + } + + // if type or options are not specified, infer using reflection + if (!propertyType) { + Log.info(`Using reflection to infer type of ${target.constructor.name}#${propertyName}`); + let reflectedType; + try { + reflectedType = + Reflect && Reflect.getMetadata + ? Reflect.getMetadata("design:type", target, propertyName) + : undefined; + propertyType = ReflectedTypeToTigrisType.get(reflectedType.name); + } catch { + throw new ReflectionNotEnabled(target, propertyName); + } + + // if propertyType is Array, subtype is required + if (propertyType === TigrisDataTypes.ARRAY && embedOptions?.elements === undefined) { + throw new IncompleteArrayTypeDefError(target, propertyName); + } + + // if propertyType is still undefined, it probably is a typed object + if (propertyType === undefined) { + propertyType = TigrisDataTypes.OBJECT; + embedOptions = { elements: reflectedType }; + } + } + + if (!propertyType) { + throw new CannotInferFieldTypeError(target, propertyName); + } + + // if propertyType is Array, subtype is required + if (propertyType === TigrisDataTypes.ARRAY && embedOptions?.elements === undefined) { + throw new IncompleteArrayTypeDefError(target, propertyName); + } + + getDecoratorMetaStorage().fields.push({ + name: propertyName, + type: propertyType, + isArray: propertyType === TigrisDataTypes.ARRAY, + target: target.constructor, + embedType: embedOptions?.elements, + arrayDepth: embedOptions?.depth, + schemaFieldOptions: fieldOptions, + } as FieldMetadata); + }; +} + +const ReflectedTypeToTigrisType: Map = new Map([ + ["String", TigrisDataTypes.STRING], + ["Boolean", TigrisDataTypes.BOOLEAN], + ["Object", TigrisDataTypes.OBJECT], + ["Array", TigrisDataTypes.ARRAY], + ["Set", TigrisDataTypes.ARRAY], + ["Number", TigrisDataTypes.NUMBER], + ["BigInt", TigrisDataTypes.NUMBER_BIGINT], +]); + +function isEmbeddedOption( + options: TigrisFieldOptions | EmbeddedFieldOptions +): options is EmbeddedFieldOptions { + return (options as EmbeddedFieldOptions).elements !== undefined; +} diff --git a/src/decorators/tigris-primary-key.ts b/src/decorators/tigris-primary-key.ts new file mode 100644 index 0000000..622335d --- /dev/null +++ b/src/decorators/tigris-primary-key.ts @@ -0,0 +1,82 @@ +import "reflect-metadata"; +import { PrimaryKeyOptions, TigrisDataTypes } from "../types"; +import { + CannotInferFieldTypeError, + IncompletePrimaryKeyDefError, + ReflectionNotEnabled, +} from "../error"; +import { getDecoratorMetaStorage } from "../globals"; +import { PrimaryKeyMetadata } from "./metadata/primary-key-metadata"; +import { Log } from "../utils/logger"; + +/** + * PrimaryKey decorator is used to mark a class property as Primary Key in a collection. + * + * Uses `Reflection` to determine the data type of schema Field + * + * @param options - Additional properties + */ +export function PrimaryKey(options: PrimaryKeyOptions): PropertyDecorator; +/** + * PrimaryKey decorator is used to mark a class property as Primary Key in a collection. + * + * Uses `Reflection` to determine the type of schema Field + * + * @param type - Schema field's data type + * @param options - Additional properties + */ +export function PrimaryKey(type: TigrisDataTypes, options: PrimaryKeyOptions): PropertyDecorator; + +/** + * PrimaryKey decorator is used to mark a class property as Primary Key in a collection. + */ +export function PrimaryKey( + typeOrOptions: TigrisDataTypes | PrimaryKeyOptions, + options?: PrimaryKeyOptions +): PropertyDecorator { + return function (target, propertyName) { + propertyName = propertyName.toString(); + let propertyType: TigrisDataTypes; + + if (typeof typeOrOptions === "string") { + propertyType = typeOrOptions as TigrisDataTypes; + } else if (typeof typeOrOptions === "object") { + options = typeOrOptions as PrimaryKeyOptions; + } + + // throw error if options are undefined + if (!options) { + throw new IncompletePrimaryKeyDefError(target, propertyName); + } + + // infer type from reflection + if (!propertyType) { + Log.info(`Using reflection to infer type of ${target.constructor.name}#${propertyName}`); + try { + const reflectedType = + Reflect && Reflect.getMetadata + ? Reflect.getMetadata("design:type", target, propertyName) + : undefined; + propertyType = ReflectedTypeToTigrisType.get(reflectedType.name); + } catch { + throw new ReflectionNotEnabled(target, propertyName); + } + } + if (!propertyType) { + throw new CannotInferFieldTypeError(target, propertyName); + } + + getDecoratorMetaStorage().primaryKeys.push({ + name: propertyName, + type: propertyType, + target: target.constructor, + options: options, + } as PrimaryKeyMetadata); + }; +} + +const ReflectedTypeToTigrisType: Map = new Map([ + ["String", TigrisDataTypes.STRING], + ["Number", TigrisDataTypes.NUMBER], + ["BigInt", TigrisDataTypes.NUMBER_BIGINT], +]); diff --git a/src/error.ts b/src/error.ts index ceb33ef..5d79f44 100644 --- a/src/error.ts +++ b/src/error.ts @@ -54,3 +54,48 @@ export class TigrisMoreThanOneSchemaDefined extends TigrisError { return "TigrisMoreThanOneSchemaDefined"; } } + +export class ReflectionNotEnabled extends TigrisError { + constructor(object: Object, propertyName: string) { + super( + `Cannot infer property 'type' for ${object.constructor.name}#${propertyName} using Reflection. + Ensure that 'emitDecoratorMetadata' option is set to true in 'tsconfig.json'. Also, make sure + to import 'reflect-metadata' on top of the main entry file in application` + ); + } + + override get name(): string { + return "ReflectionNotEnabled"; + } +} + +export class CannotInferFieldTypeError extends TigrisError { + constructor(object: Object, propertyName: string) { + super(`Field type for ${object.constructor.name}#${propertyName} cannot be determined`); + } + + override get name(): string { + return "CannotInferFieldTypeError"; + } +} + +export class IncompleteArrayTypeDefError extends TigrisError { + constructor(object: Object, propertyName: string) { + super( + `Missing "EmbeddedFieldOptions". Array's item type for ${object.constructor.name}#${propertyName} cannot be determined` + ); + } + override get name(): string { + return "IncompleteArrayTypeDefError"; + } +} + +export class IncompletePrimaryKeyDefError extends TigrisError { + constructor(object: Object, propertyName: string) { + super(`Missing "PrimaryKeyOptions" for ${object.constructor.name}#${propertyName}`); + } + + override get name(): string { + return "IncompletePrimaryKeyDefError"; + } +} diff --git a/src/globals.ts b/src/globals.ts new file mode 100644 index 0000000..c19ef8b --- /dev/null +++ b/src/globals.ts @@ -0,0 +1,9 @@ +import { DecoratorMetaStorage } from "./decorators/metadata/decorator-meta-storage"; + +export function getDecoratorMetaStorage(): DecoratorMetaStorage { + if (!global.annotationCache) { + global.annotationCache = new DecoratorMetaStorage(); + } + + return global.annotationCache; +} diff --git a/src/index.ts b/src/index.ts index 9f688bb..15b515d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,3 +3,9 @@ export * from "./db"; export * from "./session"; export * from "./tigris"; export * from "./types"; +export * from "./search/types"; +export { Field } from "./decorators/tigris-field"; +export { PrimaryKey } from "./decorators/tigris-primary-key"; +export { TigrisCollection } from "./decorators/tigris-collection"; +export { EmbeddedFieldOptions } from "./decorators/options/embedded-field-options"; +export { Cursor } from "./consumables/cursor"; diff --git a/src/schema/decorated-schema-processor.ts b/src/schema/decorated-schema-processor.ts new file mode 100644 index 0000000..8f410cb --- /dev/null +++ b/src/schema/decorated-schema-processor.ts @@ -0,0 +1,104 @@ +import { DecoratorMetaStorage } from "../decorators/metadata/decorator-meta-storage"; +import { getDecoratorMetaStorage } from "../globals"; +import { TigrisCollectionType, TigrisDataTypes, TigrisSchema } from "../types"; + +export type CollectionSchema = { + name: string; + schema: TigrisSchema; +}; + +/** @internal */ +export class DecoratedSchemaProcessor { + private static _instance: DecoratedSchemaProcessor; + readonly storage: DecoratorMetaStorage; + + private constructor() { + this.storage = getDecoratorMetaStorage(); + } + + static get Instance(): DecoratedSchemaProcessor { + if (!DecoratedSchemaProcessor._instance) { + DecoratedSchemaProcessor._instance = new DecoratedSchemaProcessor(); + } + return DecoratedSchemaProcessor._instance; + } + + process(cls: new () => TigrisCollectionType): CollectionSchema { + const collection = this.storage.filterCollectionByTarget(cls); + const schema = this.buildTigrisSchema(collection.target); + this.addPrimaryKeys(schema, collection.target); + return { + name: collection.collectionName, + schema: schema as TigrisSchema, + }; + } + + private buildTigrisSchema(from: Function): TigrisSchema { + const schema: TigrisSchema = {}; + // get all top level fields matching this target + for (const field of this.storage.filterFieldsByTarget(from)) { + const key = field.name; + schema[key] = { type: field.type }; + let arrayItems: Object, arrayDepth: number; + + switch (field.type) { + case TigrisDataTypes.ARRAY: + arrayItems = + typeof field.embedType === "function" + ? { type: this.buildTigrisSchema(field.embedType as Function) } + : { type: field.embedType as TigrisDataTypes }; + arrayDepth = field.arrayDepth && field.arrayDepth > 1 ? field.arrayDepth : 1; + schema[key] = this.buildNestedArray(arrayItems, arrayDepth); + break; + case TigrisDataTypes.OBJECT: + if (typeof field.embedType === "function") { + const embedSchema = this.buildTigrisSchema(field.embedType as Function); + // generate embedded schema as its a class + if (Object.keys(embedSchema).length > 0) { + schema[key] = { type: this.buildTigrisSchema(field.embedType as Function) }; + } + } + break; + case TigrisDataTypes.BYTE_STRING: + case TigrisDataTypes.STRING: + if (field.schemaFieldOptions?.maxLength) { + schema[key].maxLength = field.schemaFieldOptions.maxLength; + } + break; + } + } + return schema; + } + + private buildNestedArray(items, depth: number) { + let head: Object, prev: Object, next: Object; + while (depth > 0) { + if (!head) { + next = {}; + head = next; + } + next["type"] = TigrisDataTypes.ARRAY; + next["items"] = {}; + prev = next; + next = next["items"]; + depth -= 1; + } + prev["items"] = items; + return head; + } + + private addPrimaryKeys( + targetSchema: TigrisSchema, + collectionClass: Function + ) { + for (const pk of this.storage.filterPKsByTarget(collectionClass)) { + targetSchema[pk.name] = { + type: pk.type, + primary_key: { + order: pk.options?.order, + autoGenerate: pk.options.autoGenerate === true, + }, + }; + } + } +} diff --git a/src/types.ts b/src/types.ts index a899c3c..5f87c58 100644 --- a/src/types.ts +++ b/src/types.ts @@ -387,10 +387,14 @@ export enum TigrisDataTypes { OBJECT = "object", } +export interface TigrisFieldOptions { + maxLength?: number; +} + export type TigrisSchema = { [K in keyof T]: { type: TigrisDataTypes | TigrisSchema; - primary_key?: TigrisPrimaryKey; + primary_key?: PrimaryKeyOptions; items?: TigrisArrayItem; }; }; @@ -400,7 +404,7 @@ export type TigrisArrayItem = { items?: TigrisArrayItem | TigrisDataTypes; }; -export type TigrisPrimaryKey = { +export type PrimaryKeyOptions = { order: number; autoGenerate?: boolean; }; diff --git a/tsconfig.json b/tsconfig.json index 3ebf75e..f6c569a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "experimentalDecorators": true, + "emitDecoratorMetadata": true, "target": "es6", "module": "commonjs", "declaration": true, From d2e6f5e28b5fe1e6c2c254355543702964151e85 Mon Sep 17 00:00:00 2001 From: Adil Ansari Date: Fri, 16 Dec 2022 09:29:14 -0800 Subject: [PATCH 13/15] feat: Register Schemas api for decorated classes (#184) * feat: Sync collection api on Tigris client for decorated classes * Removing manifest loader * create/get/drop collection to accept class as input * Changing method name to registerSchemas --- package-lock.json | 14 -- package.json | 1 - src/__tests__/consumables/cursor.spec.ts | 4 +- .../data/invalidModels/multiExport/users.ts | 77 ---------- src/__tests__/data/models/catalog/orders.ts | 1 - src/__tests__/data/models/catalog/products.ts | 22 --- src/__tests__/data/models/embedded/users.ts | 77 ---------- src/__tests__/data/models/empty/.gitkeep | 0 src/__tests__/tigris.rpc.spec.ts | 13 +- src/__tests__/utils/manifest-loader.spec.ts | 136 ------------------ src/consumables/abstract-cursor.ts | 4 +- src/db.ts | 62 +++++++- .../metadata/decorator-meta-storage.ts | 14 +- src/error.ts | 53 +++---- src/schema/decorated-schema-processor.ts | 8 +- src/tigris.ts | 105 ++++++++++---- src/utils/manifest-loader.ts | 103 ------------- 17 files changed, 185 insertions(+), 509 deletions(-) delete mode 100644 src/__tests__/data/invalidModels/multiExport/users.ts delete mode 100644 src/__tests__/data/models/catalog/orders.ts delete mode 100644 src/__tests__/data/models/catalog/products.ts delete mode 100644 src/__tests__/data/models/embedded/users.ts delete mode 100644 src/__tests__/data/models/empty/.gitkeep delete mode 100644 src/__tests__/utils/manifest-loader.spec.ts delete mode 100644 src/utils/manifest-loader.ts diff --git a/package-lock.json b/package-lock.json index 560d10c..4564262 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.6.10", - "app-root-path": "^3.1.0", "chalk": "4.1.2", "dotenv": "^16.0.3", "google-protobuf": "^3.21.0", @@ -2310,14 +2309,6 @@ "node": ">= 8" } }, - "node_modules/app-root-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", - "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -13188,11 +13179,6 @@ "picomatch": "^2.0.4" } }, - "app-root-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", - "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==" - }, "aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", diff --git a/package.json b/package.json index 82f00cb..f450616 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,6 @@ }, "dependencies": { "@grpc/grpc-js": "^1.6.10", - "app-root-path": "^3.1.0", "chalk": "4.1.2", "dotenv": "^16.0.3", "google-protobuf": "^3.21.0", diff --git a/src/__tests__/consumables/cursor.spec.ts b/src/__tests__/consumables/cursor.spec.ts index 9335f90..ef04c14 100644 --- a/src/__tests__/consumables/cursor.spec.ts +++ b/src/__tests__/consumables/cursor.spec.ts @@ -3,7 +3,7 @@ import TestService, {TestTigrisService} from "../test-service"; import {TigrisService} from "../../proto/server/v1/api_grpc_pb"; import {IBook} from "../tigris.rpc.spec"; import {Tigris} from "../../tigris"; -import {TigrisCursorInUseError} from "../../error"; +import {CursorInUseError} from "../../error"; import {ObservabilityService} from "../../proto/server/v1/observability_grpc_pb"; import TestObservabilityService from "../test-observability-service"; import {DB} from "../../db"; @@ -76,7 +76,7 @@ describe("class FindCursor", () => { const cursor = db.getCollection("books").findMany(); // cursor is backed by is a generator fn, calling next() would retrieve item from stream cursor[Symbol.asyncIterator]().next(); - expect(() => cursor.toArray()).toThrow(TigrisCursorInUseError); + expect(() => cursor.toArray()).toThrow(CursorInUseError); }) it("allows cursor to be re-used once reset", async () => { diff --git a/src/__tests__/data/invalidModels/multiExport/users.ts b/src/__tests__/data/invalidModels/multiExport/users.ts deleted file mode 100644 index 6d37fce..0000000 --- a/src/__tests__/data/invalidModels/multiExport/users.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { TigrisCollectionType, TigrisDataTypes, TigrisSchema } from "../../../../types"; - -export interface Identity { - connection: string; - isSocial: boolean; - provider: string; - user_id: string; -} - -export const identitySchema: TigrisSchema = { - connection: { - type: TigrisDataTypes.STRING, - }, - isSocial: { - type: TigrisDataTypes.BOOLEAN, - }, - provider: { - type: TigrisDataTypes.STRING, - }, - user_id: { - type: TigrisDataTypes.STRING, - }, -}; - -export interface Stat { - loginsCount: string; -} - -export const statSchema: TigrisSchema = { - loginsCount: { - type: TigrisDataTypes.INT64, - }, -}; - -export interface User extends TigrisCollectionType { - created: string; - email: string; - identities: Identity; - name: string; - picture: string; - stats: Stat; - updated: string; - user_id: string; -} - -export const userSchema: TigrisSchema = { - created: { - type: TigrisDataTypes.DATE_TIME, - }, - email: { - type: TigrisDataTypes.STRING, - }, - identities: { - type: TigrisDataTypes.ARRAY, - items: { - type: identitySchema, - }, - }, - name: { - type: TigrisDataTypes.STRING, - }, - picture: { - type: TigrisDataTypes.STRING, - }, - stats: { - type: statSchema, - }, - updated: { - type: TigrisDataTypes.DATE_TIME, - }, - user_id: { - type: TigrisDataTypes.STRING, - primary_key: { - order: 1, - }, - }, -}; diff --git a/src/__tests__/data/models/catalog/orders.ts b/src/__tests__/data/models/catalog/orders.ts deleted file mode 100644 index 5435d1d..0000000 --- a/src/__tests__/data/models/catalog/orders.ts +++ /dev/null @@ -1 +0,0 @@ -export const NotSchema = { key: "value" }; diff --git a/src/__tests__/data/models/catalog/products.ts b/src/__tests__/data/models/catalog/products.ts deleted file mode 100644 index 757ee63..0000000 --- a/src/__tests__/data/models/catalog/products.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { - TigrisCollectionType, - TigrisDataTypes, - TigrisSchema -} from '../../../../types' - -export interface Product extends TigrisCollectionType { - id?: number; - title: string; - description: string; - price: number; -} - -export const ProductSchema: TigrisSchema = { - id: { - type: TigrisDataTypes.INT32, - primary_key: { order: 1, autoGenerate: true } - }, - title: { type: TigrisDataTypes.STRING }, - description: { type: TigrisDataTypes.STRING }, - price: { type: TigrisDataTypes.NUMBER } -} diff --git a/src/__tests__/data/models/embedded/users.ts b/src/__tests__/data/models/embedded/users.ts deleted file mode 100644 index ad7c361..0000000 --- a/src/__tests__/data/models/embedded/users.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { TigrisCollectionType, TigrisDataTypes, TigrisSchema } from "../../../../types"; - -export interface Identity { - connection: string; - isSocial: boolean; - provider: string; - user_id: string; -} - -const identitySchema: TigrisSchema = { - connection: { - type: TigrisDataTypes.STRING, - }, - isSocial: { - type: TigrisDataTypes.BOOLEAN, - }, - provider: { - type: TigrisDataTypes.STRING, - }, - user_id: { - type: TigrisDataTypes.STRING, - }, -}; - -export interface Stat { - loginsCount: string; -} - -const statSchema: TigrisSchema = { - loginsCount: { - type: TigrisDataTypes.INT64, - }, -}; - -export interface User extends TigrisCollectionType { - created: string; - email: string; - identities: Identity; - name: string; - picture: string; - stats: Stat; - updated: string; - user_id: string; -} - -export const userSchema: TigrisSchema = { - created: { - type: TigrisDataTypes.DATE_TIME, - }, - email: { - type: TigrisDataTypes.STRING, - }, - identities: { - type: TigrisDataTypes.ARRAY, - items: { - type: identitySchema, - }, - }, - name: { - type: TigrisDataTypes.STRING, - }, - picture: { - type: TigrisDataTypes.STRING, - }, - stats: { - type: statSchema, - }, - updated: { - type: TigrisDataTypes.DATE_TIME, - }, - user_id: { - type: TigrisDataTypes.STRING, - primary_key: { - order: 1, - }, - }, -}; diff --git a/src/__tests__/data/models/empty/.gitkeep b/src/__tests__/data/models/empty/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index f28595c..b3a701c 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -17,6 +17,9 @@ import { Utility } from "../utility"; import { ObservabilityService } from "../proto/server/v1/observability_grpc_pb"; import TestObservabilityService from "./test-observability-service"; import { capture, spy } from "ts-mockito"; +import { TigrisCollection } from "../decorators/tigris-collection"; +import { PrimaryKey } from "../decorators/tigris-primary-key"; +import { Field } from "../decorators/tigris-field"; describe("rpc tests", () => { let server: Server; @@ -261,7 +264,7 @@ describe("rpc tests", () => { it("delete", () => { const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); const db1 = tigris.getDatabase(); - const deletionPromise = db1.getCollection("books").deleteMany({ + const deletionPromise = db1.getCollection(IBook).deleteMany({ op: SelectorFilterOperator.EQ, fields: { id: 1 @@ -662,13 +665,19 @@ describe("rpc tests", () => { }); }); -export interface IBook extends TigrisCollectionType { +@TigrisCollection("books") +export class IBook implements TigrisCollectionType { + @PrimaryKey({order: 1}) id: number; + @Field() title: string; + @Field() author: string; + @Field({elements: TigrisDataTypes.STRING}) tags?: string[]; } + export interface IBook1 extends TigrisCollectionType { id?: number; title: string; diff --git a/src/__tests__/utils/manifest-loader.spec.ts b/src/__tests__/utils/manifest-loader.spec.ts deleted file mode 100644 index 6438c4f..0000000 --- a/src/__tests__/utils/manifest-loader.spec.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { - canBeSchema, - DatabaseManifest, - loadTigrisManifest, -} from "../../utils/manifest-loader"; -import { TigrisDataTypes } from "../../types"; -import { TigrisFileNotFoundError, TigrisMoreThanOneSchemaDefined } from "../../error"; - -describe("Manifest loader", () => { - - it("generates manifest from directory with single collection", () => { - const schemaPath = process.cwd() + "/src/__tests__/data/models/catalog"; - const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); - expect(dbManifest.collections).toHaveLength(1); - - const expected: DatabaseManifest = { - "collections": [{ - "collectionName": "products", - "schema": { - "id": { "type": "int32", "primary_key": { "order": 1, "autoGenerate": true } }, - "title": { "type": "string" }, - "description": { "type": "string" }, - "price": { "type": "number" } - }, - "schemaName": "ProductSchema" - }] - }; - expect(dbManifest).toStrictEqual(expected); - }); - - it("generates manifest from directory with embedded data model", () => { - const schemaPath = process.cwd() + "/src/__tests__/data/models/embedded"; - const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); - expect(dbManifest.collections).toHaveLength(1); - - const expected: DatabaseManifest = { - "collections": [{ - "collectionName": "users", - "schemaName": "userSchema", - "schema": { - "created": { "type": "date-time" }, - "email": { "type": "string" }, - "identities": { - "type": "array", - "items": { - "type": { - "connection": { "type": "string" }, - "isSocial": { "type": "boolean" }, - "provider": { "type": "string" }, - "user_id": { "type": "string" } - } - } - }, - "name": { "type": "string" }, - "picture": { "type": "string" }, - "stats": { - "type": { - "loginsCount": { "type": "int64" } - } - }, - "updated": { "type": "date-time" }, - "user_id": { "type": "string", "primary_key": { "order": 1 } } - } - }]}; - expect(dbManifest).toStrictEqual(expected); - }); - - it("does not generate manifest from empty directory", () => { - const schemaPath = process.cwd() + "/src/__tests__/data/models/empty"; - const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); - expect(dbManifest.collections).toHaveLength(0); - - const expected: DatabaseManifest = { "collections": [] }; - expect(dbManifest).toStrictEqual(expected); - }); - - it("throws error for invalid path", () => { - const schemaPath = "/src/__tests__/data/doesNotExist"; - expect(() => loadTigrisManifest(schemaPath)).toThrow(TigrisFileNotFoundError); - }); - - it("throws error for multiple schema exports", () => { - const schemaPath = process.cwd() + "/src/__tests__/data/invalidModels/multiExport"; - expect(() => loadTigrisManifest(schemaPath)).toThrow(TigrisMoreThanOneSchemaDefined); - }); - - const validSchemaDefinitions = [ - { key: { type: "value" } }, - { - id: { - type: TigrisDataTypes.INT32, - primary_key: { - order: 1, - autoGenerate: true - } - }, - active: { type: TigrisDataTypes.BOOLEAN } - } - ]; - test.each(validSchemaDefinitions)( - "identifies valid schema definition %p", - (definition) => { - expect(canBeSchema(definition)).toBeTruthy(); - } - ); - - const invalidSchemaDefinitions = [ - { key: "value" }, - 12, - { - id: { - type: TigrisDataTypes.INT32, - primary_key: { - order: 1, - autoGenerate: true - } - }, - active: false - }, - { - id: { - key: "value" - } - }, - { type: "string" }, - undefined, - null - ]; - - test.each(invalidSchemaDefinitions)( - "identifies invalid schema definition %p", - (definition) => { - expect(canBeSchema(definition)).toBeFalsy(); - } - ); -}); diff --git a/src/consumables/abstract-cursor.ts b/src/consumables/abstract-cursor.ts index 868bc32..61d8731 100644 --- a/src/consumables/abstract-cursor.ts +++ b/src/consumables/abstract-cursor.ts @@ -1,6 +1,6 @@ import * as proto from "google-protobuf"; import { ClientReadableStream } from "@grpc/grpc-js"; -import { TigrisCursorInUseError } from "../error"; +import { CursorInUseError } from "../error"; import { Readable } from "node:stream"; /** @internal */ @@ -36,7 +36,7 @@ export abstract class AbstractCursor { /** @internal */ private _assertNotInUse() { if (this[tClosed]) { - throw new TigrisCursorInUseError(); + throw new CursorInUseError(); } this[tClosed] = true; } diff --git a/src/db.ts b/src/db.ts index 675c104..895b2ce 100644 --- a/src/db.ts +++ b/src/db.ts @@ -27,8 +27,11 @@ import { Session } from "./session"; import { Utility } from "./utility"; import { Metadata, ServiceError } from "@grpc/grpc-js"; import { TigrisClientConfig } from "./tigris"; -import { Log } from "./utils/logger"; import { DecoratedSchemaProcessor } from "./schema/decorated-schema-processor"; +import { Log } from "./utils/logger"; +import { DecoratorMetaStorage } from "./decorators/metadata/decorator-meta-storage"; +import { getDecoratorMetaStorage } from "./globals"; +import { CollectionNotFoundError } from "./error"; /** * Tigris Database @@ -42,18 +45,20 @@ export class DB { private readonly grpcClient: TigrisClient; private readonly config: TigrisClientConfig; private readonly schemaProcessor: DecoratedSchemaProcessor; + private readonly _metadataStorage: DecoratorMetaStorage; constructor(db: string, grpcClient: TigrisClient, config: TigrisClientConfig) { this._db = db; this.grpcClient = grpcClient; this.config = config; this.schemaProcessor = DecoratedSchemaProcessor.Instance; + this._metadataStorage = getDecoratorMetaStorage(); } /** * Create a new collection if not exists. Else, apply schema changes, if any. * - * @param cls - A Class representing schema fields using decorators + * @param cls - A Class decorated by {@link TigrisCollection} * * @example * @@ -131,13 +136,13 @@ export class DB { ): Promise { return new Promise((resolve, reject) => { const rawJSONSchema: string = Utility._toJSONSchema(name, schema); - Log.debug(rawJSONSchema); const createOrUpdateCollectionRequest = new ProtoCreateOrUpdateCollectionRequest() .setProject(this._db) .setCollection(name) .setOnlyCreate(false) .setSchema(Utility.stringToUint8Array(rawJSONSchema)); + Log.event(`Creating collection: '${name}' in project: '${this._db}'`); this.grpcClient.createOrUpdateCollection( createOrUpdateCollectionRequest, // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -174,7 +179,23 @@ export class DB { }); } - public dropCollection(collectionName: string): Promise { + /** + * Drops a {@link Collection} + * + * @param cls - A Class decorated by {@link TigrisCollection} + */ + public dropCollection(cls: new () => TigrisCollectionType): Promise; + /** + * Drops a {@link Collection} + * + * @param name - Collection name + */ + public dropCollection(name: string): Promise; + + public dropCollection( + nameOrClass: TigrisCollectionType | string + ): Promise { + const collectionName = this.resolveNameFromCollectionClass(nameOrClass); return new Promise((resolve, reject) => { this.grpcClient.dropCollection( new ProtoDropCollectionRequest().setProject(this.db).setCollection(collectionName), @@ -227,10 +248,41 @@ export class DB { }); } - public getCollection(collectionName: string): Collection { + /** + * Gets a {@link Collection} object + * + * @param cls - A Class decorated by {@link TigrisCollection} + */ + public getCollection( + cls: new () => TigrisCollectionType + ): Collection; + /** + * Gets a {@link Collection} object + * + * @param name - Collection name + */ + public getCollection(name: string): Collection; + public getCollection(nameOrClass: T | string): Collection { + const collectionName = this.resolveNameFromCollectionClass(nameOrClass); return new Collection(collectionName, this.db, this.grpcClient, this.config); } + private resolveNameFromCollectionClass(nameOrClass: TigrisCollectionType | string) { + let collectionName: string; + if (typeof nameOrClass === "string") { + collectionName = nameOrClass; + } else { + const coll = this._metadataStorage.getCollectionByTarget( + nameOrClass as new () => TigrisCollectionType + ); + if (!coll) { + throw new CollectionNotFoundError(nameOrClass.toString()); + } + collectionName = coll.collectionName; + } + return collectionName; + } + public transact(fn: (tx: Session) => void): Promise { return new Promise((resolve, reject) => { this.beginTransaction() diff --git a/src/decorators/metadata/decorator-meta-storage.ts b/src/decorators/metadata/decorator-meta-storage.ts index a178247..66d7ab5 100644 --- a/src/decorators/metadata/decorator-meta-storage.ts +++ b/src/decorators/metadata/decorator-meta-storage.ts @@ -14,7 +14,15 @@ export class DecoratorMetaStorage { readonly fields: Array = new Array(); readonly primaryKeys: Array = new Array(); - filterCollectionByTarget(target: Function): CollectionMetadata { + getAllCollections(): IterableIterator { + return this.collections.values(); + } + + getCollectionByName(name: string): CollectionMetadata { + return this.collections.get(name); + } + + getCollectionByTarget(target: Function): CollectionMetadata { for (const collection of this.collections.values()) { if (collection.target === target) { return collection; @@ -22,13 +30,13 @@ export class DecoratorMetaStorage { } } - filterFieldsByTarget(target: Function): FieldMetadata[] { + getFieldsByTarget(target: Function): Array { return this.fields.filter(function (field) { return field.target === target; }); } - filterPKsByTarget(target: Function): PrimaryKeyMetadata[] { + getPKsByTarget(target: Function): Array { return this.primaryKeys.filter(function (pk) { return pk.target === target; }); diff --git a/src/error.ts b/src/error.ts index 5d79f44..502b540 100644 --- a/src/error.ts +++ b/src/error.ts @@ -18,49 +18,22 @@ export class TigrisError extends Error { * @public * @category Error */ -export class TigrisCursorInUseError extends TigrisError { +export class CursorInUseError extends TigrisError { constructor(message = "Cursor is already in use or used. Please reset()") { super(message); } override get name(): string { - return "TigrisCursorInUseError"; - } -} - -/** - * An error thrown when path is invalid or not found - * - * @public - * @category Error - */ -export class TigrisFileNotFoundError extends TigrisError { - constructor(message) { - super(message); - } - - override get name(): string { - return "TigrisFileNotFoundError"; - } -} - -export class TigrisMoreThanOneSchemaDefined extends TigrisError { - constructor(fileName, foundSchemas) { - super( - `${foundSchemas} TigrisSchema detected in file ${fileName}, should only have 1 TigrisSchema exported` - ); - } - override get name(): string { - return "TigrisMoreThanOneSchemaDefined"; + return "CursorInUseError"; } } export class ReflectionNotEnabled extends TigrisError { constructor(object: Object, propertyName: string) { super( - `Cannot infer property 'type' for ${object.constructor.name}#${propertyName} using Reflection. - Ensure that 'emitDecoratorMetadata' option is set to true in 'tsconfig.json'. Also, make sure - to import 'reflect-metadata' on top of the main entry file in application` + `Cannot infer property "type" for ${object.constructor.name}#${propertyName} using Reflection. + Ensure that "emitDecoratorMetadata" option is set to true in "tsconfig.json". Also, make sure + to "import 'reflect-metadata'" on top of the main entry file in application` ); } @@ -71,7 +44,7 @@ export class ReflectionNotEnabled extends TigrisError { export class CannotInferFieldTypeError extends TigrisError { constructor(object: Object, propertyName: string) { - super(`Field type for ${object.constructor.name}#${propertyName} cannot be determined`); + super(`Field type for '${object.constructor.name}#${propertyName}' cannot be determined`); } override get name(): string { @@ -82,7 +55,7 @@ export class CannotInferFieldTypeError extends TigrisError { export class IncompleteArrayTypeDefError extends TigrisError { constructor(object: Object, propertyName: string) { super( - `Missing "EmbeddedFieldOptions". Array's item type for ${object.constructor.name}#${propertyName} cannot be determined` + `Missing "EmbeddedFieldOptions". Array's item type for '${object.constructor.name}#${propertyName}' cannot be determined` ); } override get name(): string { @@ -92,10 +65,20 @@ export class IncompleteArrayTypeDefError extends TigrisError { export class IncompletePrimaryKeyDefError extends TigrisError { constructor(object: Object, propertyName: string) { - super(`Missing "PrimaryKeyOptions" for ${object.constructor.name}#${propertyName}`); + super(`Missing "PrimaryKeyOptions" for '${object.constructor.name}#${propertyName}'`); } override get name(): string { return "IncompletePrimaryKeyDefError"; } } + +export class CollectionNotFoundError extends TigrisError { + constructor(name: string) { + super(`Collection not found : '${name}'`); + } + + override get name(): string { + return "CollectionNotFoundError"; + } +} diff --git a/src/schema/decorated-schema-processor.ts b/src/schema/decorated-schema-processor.ts index 8f410cb..e7e1239 100644 --- a/src/schema/decorated-schema-processor.ts +++ b/src/schema/decorated-schema-processor.ts @@ -10,7 +10,7 @@ export type CollectionSchema = { /** @internal */ export class DecoratedSchemaProcessor { private static _instance: DecoratedSchemaProcessor; - readonly storage: DecoratorMetaStorage; + private readonly storage: DecoratorMetaStorage; private constructor() { this.storage = getDecoratorMetaStorage(); @@ -24,7 +24,7 @@ export class DecoratedSchemaProcessor { } process(cls: new () => TigrisCollectionType): CollectionSchema { - const collection = this.storage.filterCollectionByTarget(cls); + const collection = this.storage.getCollectionByTarget(cls); const schema = this.buildTigrisSchema(collection.target); this.addPrimaryKeys(schema, collection.target); return { @@ -36,7 +36,7 @@ export class DecoratedSchemaProcessor { private buildTigrisSchema(from: Function): TigrisSchema { const schema: TigrisSchema = {}; // get all top level fields matching this target - for (const field of this.storage.filterFieldsByTarget(from)) { + for (const field of this.storage.getFieldsByTarget(from)) { const key = field.name; schema[key] = { type: field.type }; let arrayItems: Object, arrayDepth: number; @@ -91,7 +91,7 @@ export class DecoratedSchemaProcessor { targetSchema: TigrisSchema, collectionClass: Function ) { - for (const pk of this.storage.filterPKsByTarget(collectionClass)) { + for (const pk of this.storage.getPKsByTarget(collectionClass)) { targetSchema[pk.name] = { type: pk.type, primary_key: { diff --git a/src/tigris.ts b/src/tigris.ts index 8fbd4c3..6dbae65 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -7,10 +7,8 @@ import { CreateProjectRequest as ProtoCreateProjectRequest } from "./proto/serve import { GetInfoRequest as ProtoGetInfoRequest } from "./proto/server/v1/observability_pb"; import { HealthCheckInput as ProtoHealthCheckInput } from "./proto/server/v1/health_pb"; -import path from "node:path"; -import appRootPath from "app-root-path"; import * as dotenv from "dotenv"; -import { DatabaseOptions, ServerMetadata } from "./types"; +import { DatabaseOptions, ServerMetadata, TigrisCollectionType } from "./types"; import { GetAccessTokenRequest as ProtoGetAccessTokenRequest, @@ -20,8 +18,10 @@ import { import { DB } from "./db"; import { AuthClient } from "./proto/server/v1/auth_grpc_pb"; import { Utility } from "./utility"; -import { loadTigrisManifest, DatabaseManifest } from "./utils/manifest-loader"; import { Log } from "./utils/logger"; +import { DecoratorMetaStorage } from "./decorators/metadata/decorator-meta-storage"; +import { getDecoratorMetaStorage } from "./globals"; +import { CollectionMetadata } from "./decorators/metadata/collection-metadata"; const AuthorizationHeaderName = "authorization"; const AuthorizationBearer = "Bearer "; @@ -128,6 +128,7 @@ export class Tigris { private readonly observabilityClient: ObservabilityClient; private readonly healthAPIClient: HealthAPIClient; private readonly _config: TigrisClientConfig; + private readonly _metadataStorage: DecoratorMetaStorage; private readonly _ping: () => void; private readonly pingId: NodeJS.Timeout | number | string | undefined; @@ -236,6 +237,7 @@ export class Tigris { }); } } + this._metadataStorage = getDecoratorMetaStorage(); Log.info(`Using Tigris at: ${config.serverUrl}`); } @@ -256,32 +258,85 @@ export class Tigris { } /** - * Automatically provision Databases and Collections based on the directories - * and {@link TigrisSchema} definitions in file system + * Automatically create Project and create or update Collections. + * Collection classes decorated with {@link TigrisCollection} decorator will be + * created if not already existing. If Collection already exists, schema changes + * will be applied, if any. + */ + public async registerSchemas(); + /** + * Automatically create Project and create or update Collections. + * Collection classes decorated with {@link TigrisCollection} decorator will be + * created if not already existing. If Collection already exists, schema changes + * will be applied, if any. + * + * @param collectionNames - Array of collection names as strings to be created + * or updated + * + * @example + * + * ``` + * @TigrisCollection("todoItems") + * class TodoItem implements TigrisCollectionType { + * @PrimaryKey(TigrisDataTypes.INT32, { order: 1 }) + * id: number; * - * @param schemaPath - Directory location in file system. Recommended to - * provide an absolute path, else loader will try to access application's root - * path which may not be accurate. + * @Field() + * text: string; + * } * - * @param dbName - The name of the database to create the collections in. + * await db.registerSchemas(["todoItems"]); + * ``` */ - public async registerSchemas(schemaPath: string) { - if (!path.isAbsolute(schemaPath)) { - schemaPath = path.join(appRootPath.toString(), schemaPath); - } + public async registerSchemas(collectionNames: Array); + /** + * Automatically create Project and create or update Collections. + * Collection classes decorated with {@link TigrisCollection} decorator will be + * created if not already existing. If Collection already exists, schema changes + * will be applied, if any. + * + * @param collections - Array of Collection classes + * + * @example + * ``` + * @TigrisCollection("todoItems") + * class TodoItem implements TigrisCollectionType { + * @PrimaryKey(TigrisDataTypes.INT32, { order: 1 }) + * id: number; + * + * @Field() + * text: string; + * } + * + * await db.registerSchemas([TodoItem]); + * ``` + */ + public async registerSchemas(collections: Array); + public async registerSchemas(filter?: Array) { + const projectName = this._config.projectName; + const tigrisDb = await this.createDatabaseIfNotExists(projectName); + const needUpdate: Array = new Array(); - // create DB - const dbName = this._config.projectName; - const tigrisDb = await this.createDatabaseIfNotExists(dbName); - Log.event(`Created database: ${dbName}`); + if (!filter) { + for (const coll of this._metadataStorage.getAllCollections()) { + needUpdate.push(coll); + } + } else { + for (const name of filter) { + const found = + typeof name === "string" + ? this._metadataStorage.getCollectionByName(name) + : this._metadataStorage.getCollectionByTarget(name as Function); + if (!found) { + Log.error(`No such collection defined: '${name.toString()}'`); + } else { + needUpdate.push(found); + } + } + } - const dbManifest: DatabaseManifest = loadTigrisManifest(schemaPath); - for (const coll of dbManifest.collections) { - // Create a collection - const collection = await tigrisDb.createOrUpdateCollection(coll.collectionName, coll.schema); - Log.event( - `Created collection: ${collection.collectionName} from schema: ${coll.schemaName} in db: ${dbName}` - ); + for (const coll of needUpdate) { + await tigrisDb.createOrUpdateCollection(coll.target.prototype.constructor); } } diff --git a/src/utils/manifest-loader.ts b/src/utils/manifest-loader.ts deleted file mode 100644 index 7ea2279..0000000 --- a/src/utils/manifest-loader.ts +++ /dev/null @@ -1,103 +0,0 @@ -import path from "node:path"; -import fs from "node:fs"; -import { Log } from "./logger"; -import { TigrisSchema } from "../types"; -import { TigrisFileNotFoundError, TigrisMoreThanOneSchemaDefined } from "../error"; - -const COLL_FILE_SUFFIX = ".ts"; - -type CollectionManifest = { - collectionName: string; - schemaName: string; - schema: TigrisSchema; -}; - -/** - * Array of collections in the database - */ -export type DatabaseManifest = { - collections: Array; -}; - -/** - * Loads the databases and schema definitions from file system that can be used - * to create databases and collections - * - * @return TigrisManifest - */ -export function loadTigrisManifest(schemasPath: string): DatabaseManifest { - Log.event(`Scanning ${schemasPath} for Tigris schema definitions`); - - if (!fs.existsSync(schemasPath)) { - Log.error(`Invalid path for Tigris schema: ${schemasPath}`); - throw new TigrisFileNotFoundError(`Directory not found: ${schemasPath}`); - } - - const dbManifest: DatabaseManifest = { - collections: new Array(), - }; - - // load manifest from file structure - for (const colsFileName of fs.readdirSync(schemasPath)) { - const collFilePath = path.join(schemasPath, colsFileName); - if (collFilePath.endsWith(COLL_FILE_SUFFIX) && fs.lstatSync(collFilePath).isFile()) { - Log.info(`Found Schema file ${colsFileName} in ${schemasPath}`); - const collName = colsFileName.slice( - 0, - Math.max(0, colsFileName.length - COLL_FILE_SUFFIX.length) - ); - - // eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module - const schemaFile = require(collFilePath); - const detectedSchemas = new Map>(); - - // read schemas in that file - for (const [key, value] of Object.entries(schemaFile)) { - if (canBeSchema(value)) { - detectedSchemas.set(key, value as TigrisSchema); - } - } - - if (detectedSchemas.size > 1) { - throw new TigrisMoreThanOneSchemaDefined(collFilePath, detectedSchemas.size); - } - - for (const [name, def] of detectedSchemas) { - dbManifest.collections.push({ - collectionName: collName, - schema: def, - schemaName: name, - }); - Log.info(`Found schema definition: ${name}`); - } - } - } - - if (dbManifest.collections.length === 0) { - Log.warn(`No valid schema definition found in ${schemasPath}`); - } - - Log.debug(`Generated DB Manifest: ${JSON.stringify(dbManifest)}`); - return dbManifest; -} - -/** - * Validate if given input can be a valid {@link TigrisSchema} type. This is - * not a comprehensive validation, it happens on server. - * - * @param maybeSchema - */ -export function canBeSchema(maybeSchema: unknown): boolean { - if (maybeSchema === null || typeof maybeSchema !== "object") { - return false; - } - for (const value of Object.values(maybeSchema)) { - if (value === null || typeof value !== "object") { - return false; - } - if (!Object.prototype.hasOwnProperty.call(value, "type")) { - return false; - } - } - return true; -} From 7261e62c5977cd5fefbb38342d39e42832e4265b Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Fri, 16 Dec 2022 09:37:39 -0800 Subject: [PATCH 14/15] fix: Fix detecting project name --- src/__tests__/tigris.rpc.spec.ts | 2 +- src/tigris.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index b3a701c..329fb6d 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -656,7 +656,7 @@ describe("rpc tests", () => { }); it("serverMetadata", () => { - const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT}); + const tigris = new Tigris({serverUrl: "localhost:" + SERVER_PORT, projectName: "db3"}); const serverMetadataPromise = tigris.getServerMetadata(); serverMetadataPromise.then(value => { expect(value.serverVersion).toBe("1.0.0-test-service"); diff --git a/src/tigris.ts b/src/tigris.ts index 6dbae65..771ed99 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -143,11 +143,6 @@ export class Tigris { } if (config.serverUrl === undefined) { config.serverUrl = DEFAULT_URL; - if (!("TIGRIS_PROJECT" in process.env)) { - throw new Error("Unable to resolve TIGRIS_PROJECT environment variable"); - } else { - config.projectName = process.env.TIGRIS_PROJECT; - } if (process.env.TIGRIS_URI?.trim().length > 0) { config.serverUrl = process.env.TIGRIS_URI; } @@ -156,6 +151,14 @@ export class Tigris { } } + if (config.projectName === undefined) { + if (!("TIGRIS_PROJECT" in process.env)) { + throw new Error("Unable to resolve TIGRIS_PROJECT environment variable"); + } + + config.projectName = process.env.TIGRIS_PROJECT; + } + if (config.serverUrl.startsWith("https://")) { config.serverUrl = config.serverUrl.replace("https://", ""); } From 98d2581c5758b71f0ce6d60414f43024cf331d58 Mon Sep 17 00:00:00 2001 From: Jigar Joshi Date: Tue, 20 Dec 2022 09:57:37 -0800 Subject: [PATCH 15/15] fix: Upgraded API (#188) --- api/proto | 2 +- src/__tests__/test-service.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/proto b/api/proto index d9d2608..9aac840 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit d9d2608f582edfa9dc95f89d169b5c0d26e68360 +Subproject commit 9aac840b172800bc528ad12ee1c634d53b4b14ae diff --git a/src/__tests__/test-service.ts b/src/__tests__/test-service.ts index 0f40e35..fe6ea86 100644 --- a/src/__tests__/test-service.ts +++ b/src/__tests__/test-service.ts @@ -95,7 +95,9 @@ export class TestTigrisService { } } - public impl: ITigrisServer = { + public impl: { + deleteBranch(call: ServerUnaryCall, callback: sendUnaryData): void; read(call: ServerWritableStream): void; describeDatabase(call: ServerUnaryCall, callback: sendUnaryData): void; replace(call: ServerUnaryCall, callback: sendUnaryData): void; rollbackTransaction(call: ServerUnaryCall, callback: sendUnaryData): void; insert(call: ServerUnaryCall, callback: sendUnaryData): void; update(call: ServerUnaryCall, callback: sendUnaryData): void; createProject(call: ServerUnaryCall, callback: sendUnaryData): void; listProjects(call: ServerUnaryCall, callback: sendUnaryData): void; delete(call: ServerUnaryCall, callback: sendUnaryData): void; describeCollection(_call: ServerUnaryCall, _callback: sendUnaryData): void; search(call: ServerWritableStream): void; createOrUpdateCollection(call: ServerUnaryCall, callback: sendUnaryData): void; beginTransaction(call: ServerUnaryCall, callback: sendUnaryData): void; commitTransaction(call: ServerUnaryCall, callback: sendUnaryData): void; dropCollection(call: ServerUnaryCall, callback: sendUnaryData): void; deleteProject(call: ServerUnaryCall, callback: sendUnaryData): void; createBranch(call: ServerUnaryCall, callback: sendUnaryData): void; listCollections(call: ServerUnaryCall, callback: sendUnaryData): void + } = { createBranch( call: ServerUnaryCall, callback: sendUnaryData