From 8d05ecbbcf857fcacfba1dda6e96f09a7fe3b65b Mon Sep 17 00:00:00 2001 From: Adil Ansari Date: Fri, 13 Jan 2023 17:51:36 -0800 Subject: [PATCH] feat: Database branching (#208) --- package-lock.json | 14 + package.json | 3 +- src/__tests__/consumables/cursor.spec.ts | 15 +- src/__tests__/test-service.ts | 128 ++++++-- src/__tests__/tigris.rpc.spec.ts | 357 ++++++++++++++++------- src/__tests__/tigris.utility.spec.ts | 86 +++++- src/__tests__/utils/env-loader.spec.ts | 17 ++ src/collection.ts | 9 + src/db.ts | 162 +++++++++- src/error.ts | 15 + src/session.ts | 11 +- src/tigris.ts | 14 +- src/types.ts | 69 +++-- src/utility.ts | 36 +++ src/utils/env-loader.ts | 67 +++++ 15 files changed, 823 insertions(+), 180 deletions(-) create mode 100644 src/__tests__/utils/env-loader.spec.ts create mode 100644 src/utils/env-loader.ts diff --git a/package-lock.json b/package-lock.json index e2aee1c..5111540 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.6.10", + "app-root-path": "^3.1.0", "chalk": "4.1.2", "dotenv": "^16.0.3", "google-protobuf": "^3.21.0", @@ -2309,6 +2310,14 @@ "node": ">= 8" } }, + "node_modules/app-root-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", + "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -13179,6 +13188,11 @@ "picomatch": "^2.0.4" } }, + "app-root-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", + "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==" + }, "aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", diff --git a/package.json b/package.json index f450616..6c281ef 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "google-protobuf": "^3.21.0", "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", - "typescript": "^4.7.2" + "typescript": "^4.7.2", + "app-root-path": "^3.1.0" } } diff --git a/src/__tests__/consumables/cursor.spec.ts b/src/__tests__/consumables/cursor.spec.ts index 618dcb1..93fd18d 100644 --- a/src/__tests__/consumables/cursor.spec.ts +++ b/src/__tests__/consumables/cursor.spec.ts @@ -8,12 +8,12 @@ import { ObservabilityService } from "../../proto/server/v1/observability_grpc_p import TestObservabilityService from "../test-observability-service"; import { DB } from "../../db"; -describe("class FindCursor", () => { +describe("FindCursor", () => { let server: Server; const SERVER_PORT = 5003; let db: DB; - beforeAll((done) => { + beforeAll(async () => { server = new Server(); TestTigrisService.reset(); server.addService(TigrisService, TestService.handler.impl); @@ -30,9 +30,14 @@ describe("class FindCursor", () => { } } ); - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - db = tigris.getDatabase(); - done(); + const tigris = new Tigris({ + serverUrl: "localhost:" + SERVER_PORT, + projectName: "db3", + branch: TestTigrisService.ExpectedBranch, + }); + const dbPromise = tigris.getDatabase(); + db = await dbPromise; + return dbPromise; }); beforeEach(() => { diff --git a/src/__tests__/test-service.ts b/src/__tests__/test-service.ts index dd20ccd..3def715 100644 --- a/src/__tests__/test-service.ts +++ b/src/__tests__/test-service.ts @@ -1,4 +1,5 @@ -import { ITigrisServer, TigrisService } from "../proto/server/v1/api_grpc_pb"; +import { TigrisService } from "../proto/server/v1/api_grpc_pb"; +import * as grpc from "@grpc/grpc-js"; import { sendUnaryData, ServerUnaryCall, ServerWritableStream } from "@grpc/grpc-js"; import { v4 as uuidv4 } from "uuid"; import { @@ -9,20 +10,25 @@ import { CollectionMetadata, CommitTransactionRequest, CommitTransactionResponse, - CreateProjectRequest, - CreateProjectResponse, + CreateBranchRequest, + CreateBranchResponse, CreateOrUpdateCollectionRequest, CreateOrUpdateCollectionResponse, - ProjectInfo, + CreateProjectRequest, + CreateProjectResponse, DatabaseMetadata, + DeleteBranchRequest, + DeleteBranchResponse, + DeleteProjectRequest, + DeleteProjectResponse, DeleteRequest, DeleteResponse, DescribeCollectionRequest, DescribeCollectionResponse, + DescribeDatabaseRequest, + DescribeDatabaseResponse, DropCollectionRequest, DropCollectionResponse, - DeleteProjectRequest, - DeleteProjectResponse, FacetCount, InsertRequest, InsertResponse, @@ -31,6 +37,7 @@ import { ListProjectsRequest, ListProjectsResponse, Page, + ProjectInfo, ReadRequest, ReadResponse, ReplaceRequest, @@ -47,17 +54,14 @@ import { TransactionCtx, UpdateRequest, UpdateResponse, - DescribeDatabaseRequest, - DescribeDatabaseResponse, - CreateBranchRequest, - CreateBranchResponse, - DeleteBranchRequest, - DeleteBranchResponse, } from "../proto/server/v1/api_pb"; import * as google_protobuf_timestamp_pb from "google-protobuf/google/protobuf/timestamp_pb"; import { Utility } from "../utility"; +import { Status } from "@grpc/grpc-js/build/src/constants"; +import assert from "assert"; export class TestTigrisService { + public static readonly ExpectedBranch = "unit-tests"; private static PROJECTS: string[] = []; private static COLLECTION_MAP = new Map>(); private static txId: string; @@ -179,18 +183,62 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { - // TODO implement + let err: Partial; + const reply = new CreateBranchResponse(); + + switch (call.request.getBranch()) { + case Branch.Existing: + err = { + code: Status.ALREADY_EXISTS, + details: `branch already exists '${Branch.Existing}'`, + }; + break; + case Branch.NotFound: + err = { + code: Status.NOT_FOUND, + details: `project not found`, + }; + break; + default: + reply.setStatus("created"); + reply.setMessage("branch successfully created"); + } + + if (err) { + return callback(err, undefined); + } else { + return callback(undefined, reply); + } }, deleteBranch( call: ServerUnaryCall, callback: sendUnaryData ): void { - // TODO implement + let err: Partial; + const reply = new DeleteBranchResponse(); + switch (call.request.getBranch()) { + case Branch.NotFound: + err = { + code: Status.NOT_FOUND, + details: `Branch doesn't exist`, + }; + break; + default: + reply.setStatus("deleted"); + reply.setMessage("branch deleted successfully"); + } + if (err) { + return callback(err, undefined); + } else { + return callback(undefined, reply); + } }, beginTransaction( call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const reply: BeginTransactionResponse = new BeginTransactionResponse(); if (call.request.getProject() === "test-tx") { TestTigrisService.txId = uuidv4(); @@ -204,11 +252,12 @@ export class TestTigrisService { reply.setTxCtx(new TransactionCtx().setId("id-test").setOrigin("origin-test")); callback(undefined, reply); }, - // eslint-disable-next-line @typescript-eslint/no-empty-function commitTransaction( call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const reply: CommitTransactionResponse = new CommitTransactionResponse(); reply.setStatus("committed-test"); callback(undefined, reply); @@ -223,23 +272,23 @@ export class TestTigrisService { reply.setStatus("created"); callback(undefined, reply); }, - /* eslint-disable @typescript-eslint/no-empty-function */ createOrUpdateCollection( - // eslint-disable-next-line @typescript-eslint/no-unused-vars call: ServerUnaryCall, - // eslint-disable-next-line @typescript-eslint/no-unused-vars callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const reply: CreateOrUpdateCollectionResponse = new CreateOrUpdateCollectionResponse(); reply.setStatus("Collections created successfully"); reply.setStatus(call.request.getCollection()); callback(undefined, reply); }, - /* eslint-enable @typescript-eslint/no-empty-function */ delete( call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); @@ -259,13 +308,12 @@ export class TestTigrisService { }, /* eslint-disable @typescript-eslint/no-empty-function */ describeCollection( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - _call: ServerUnaryCall, + call: ServerUnaryCall, // eslint-disable-next-line @typescript-eslint/no-unused-vars _callback: sendUnaryData - ): void {}, - - /* eslint-enable @typescript-eslint/no-empty-function */ + ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + }, describeDatabase( call: ServerUnaryCall, callback: sendUnaryData @@ -284,7 +332,10 @@ export class TestTigrisService { .setSchema("schema" + index) ); } - result.setMetadata(new DatabaseMetadata()).setCollectionsList(collectionsDescription); + result + .setMetadata(new DatabaseMetadata()) + .setCollectionsList(collectionsDescription) + .setBranchesList(["main", "staging", TestTigrisService.ExpectedBranch]); callback(undefined, result); }, @@ -292,6 +343,8 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const newCollections = TestTigrisService.COLLECTION_MAP.get(call.request.getProject()).filter( (coll) => coll !== call.request.getCollection() ); @@ -317,6 +370,8 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); @@ -356,6 +411,8 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const reply: ListCollectionsResponse = new ListCollectionsResponse(); const collectionInfos: CollectionInfo[] = []; for ( @@ -389,8 +446,9 @@ export class TestTigrisService { reply.setProjectsList(databaseInfos); callback(undefined, reply); }, - // eslint-disable-next-line @typescript-eslint/no-empty-function read(call: ServerWritableStream): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); @@ -453,8 +511,9 @@ export class TestTigrisService { call.end(); } }, - // eslint-disable-next-line @typescript-eslint/no-empty-function search(call: ServerWritableStream): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const searchMeta = new SearchMetadata().setFound(5).setTotalPages(5); // paginated search impl @@ -500,13 +559,12 @@ export class TestTigrisService { call.end(); } }, - /* eslint-disable @typescript-eslint/no-empty-function */ replace( - // eslint-disable-next-line @typescript-eslint/no-unused-vars call: ServerUnaryCall, - // eslint-disable-next-line @typescript-eslint/no-unused-vars callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); @@ -543,15 +601,18 @@ export class TestTigrisService { call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + const reply: RollbackTransactionResponse = new RollbackTransactionResponse(); reply.setStatus("rollback-test"); callback(undefined, reply); }, - /* eslint-enable @typescript-eslint/no-empty-function */ update( call: ServerUnaryCall, callback: sendUnaryData ): void { + assert(call.request.getBranch() === TestTigrisService.ExpectedBranch); + if (call.request.getProject() === "test-tx") { const txIdHeader = call.metadata.get("Tigris-Tx-Id").toString(); const txOriginHeader = call.metadata.get("Tigris-Tx-Origin").toString(); @@ -582,3 +643,8 @@ export default { service: TigrisService, handler: new TestTigrisService(), }; + +export enum Branch { + Existing = "existing", + NotFound = "no-project", +} diff --git a/src/__tests__/tigris.rpc.spec.ts b/src/__tests__/tigris.rpc.spec.ts index a36f936..e000bc8 100644 --- a/src/__tests__/tigris.rpc.spec.ts +++ b/src/__tests__/tigris.rpc.spec.ts @@ -1,6 +1,6 @@ -import { Server, ServerCredentials } from "@grpc/grpc-js"; -import { TigrisService } from "../proto/server/v1/api_grpc_pb"; -import TestService, { TestTigrisService } from "./test-service"; +import { Server, ServerCredentials, ServiceError } from "@grpc/grpc-js"; +import { TigrisClient, TigrisService } from "../proto/server/v1/api_grpc_pb"; +import TestService, { Branch, TestTigrisService } from "./test-service"; import TestServiceCache, { TestCacheService } from "./test-cache-service"; import { @@ -13,21 +13,23 @@ import { UpdateFieldsOperator, UpdateQueryOptions, } from "../types"; -import { Tigris } from "../tigris"; -import { Case, Collation, SearchQuery, SearchQueryOptions, SearchResult } from "../search/types"; +import { Tigris, TigrisClientConfig } from "../tigris"; +import { Case, Collation, SearchQuery, SearchResult } from "../search/types"; import { Utility } from "../utility"; import { ObservabilityService } from "../proto/server/v1/observability_grpc_pb"; import TestObservabilityService from "./test-observability-service"; -import { capture, spy } from "ts-mockito"; +import { anything, capture, reset, spy, when } from "ts-mockito"; import { TigrisCollection } from "../decorators/tigris-collection"; import { PrimaryKey } from "../decorators/tigris-primary-key"; import { Field } from "../decorators/tigris-field"; import { SearchIterator } from "../consumables/search-iterator"; import { CacheService } from "../proto/server/v1/cache_grpc_pb"; +import { DatabaseBranchError } from "../error"; +import { Status } from "@grpc/grpc-js/build/src/constants"; describe("rpc tests", () => { let server: Server; - const SERVER_PORT = 5002; + const testConfig = { serverUrl: "localhost:" + 5002, projectName: "db1", branch: "unit-tests" }; beforeAll((done) => { server = new Server(); @@ -36,7 +38,7 @@ describe("rpc tests", () => { server.addService(CacheService, TestServiceCache.handler.impl); server.addService(ObservabilityService, TestObservabilityService.handler.impl); server.bindAsync( - "localhost:" + SERVER_PORT, + testConfig.serverUrl, // test purpose only ServerCredentials.createInsecure(), (err: Error | null) => { @@ -60,15 +62,15 @@ describe("rpc tests", () => { done(); }); - it("getDatabase", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db1" }); - const db1 = tigris.getDatabase(); + it("getDatabase", async () => { + const tigris = new Tigris(testConfig); + const db1 = await tigris.getDatabase(); expect(db1.db).toBe("db1"); }); - it("listCollections1", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db1" }); - const db1 = tigris.getDatabase(); + it("listCollections1", async () => { + const tigris = new Tigris(testConfig); + const db1 = await tigris.getDatabase(); const listCollectionPromise = db1.listCollections(); listCollectionPromise.then((value) => { @@ -82,9 +84,9 @@ describe("rpc tests", () => { return listCollectionPromise; }); - it("listCollections2", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("listCollections2", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const listCollectionPromise = db1.listCollections(); listCollectionPromise.then((value) => { @@ -98,9 +100,9 @@ describe("rpc tests", () => { return listCollectionPromise; }); - it("describeDatabase", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("describeDatabase", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const databaseDescriptionPromise = db1.describe(); databaseDescriptionPromise.then((value) => { @@ -114,9 +116,9 @@ describe("rpc tests", () => { return databaseDescriptionPromise; }); - it("dropCollection", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("dropCollection", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const dropCollectionPromise = db1.dropCollection("db3_coll_2"); dropCollectionPromise.then((value) => { @@ -126,16 +128,16 @@ describe("rpc tests", () => { return dropCollectionPromise; }); - it("getCollection", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("getCollection", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const books = db1.getCollection("books"); expect(books.collectionName).toBe("books"); }); - it("insert", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insert", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const insertionPromise = db1.getCollection("books").insertOne({ author: "author name", id: 0, @@ -148,9 +150,9 @@ describe("rpc tests", () => { return insertionPromise; }); - it("insert2", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insert2", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const insertionPromise = db1.getCollection("books").insertOne({ id: 0, title: "science book", @@ -165,9 +167,9 @@ describe("rpc tests", () => { return insertionPromise; }); - it("insert_multi_pk", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insert_multi_pk", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const insertionPromise = db1.getCollection("books-multi-pk").insertOne({ id: 0, id2: 0, @@ -184,9 +186,9 @@ describe("rpc tests", () => { return insertionPromise; }); - it("insert_multi_pk_many", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insert_multi_pk_many", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const insertionPromise = db1.getCollection("books-multi-pk").insertMany([ { id: 0, @@ -217,9 +219,9 @@ describe("rpc tests", () => { return insertionPromise; }); - it("insertWithOptionalField", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insertWithOptionalField", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const randomNumber: number = Math.floor(Math.random() * 100); // pass the random number in author field. mock server reads author and sets as the // primaryKey field. @@ -234,9 +236,9 @@ describe("rpc tests", () => { return insertionPromise; }); - it("insertOrReplace", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insertOrReplace", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const insertOrReplacePromise = db1.getCollection("books").insertOrReplaceOne({ author: "author name", id: 0, @@ -249,9 +251,9 @@ describe("rpc tests", () => { return insertOrReplacePromise; }); - it("insertOrReplaceWithOptionalField", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("insertOrReplaceWithOptionalField", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const randomNumber: number = Math.floor(Math.random() * 100); // pass the random number in author field. mock server reads author and sets as the // primaryKey field. @@ -268,21 +270,23 @@ describe("rpc tests", () => { return insertOrReplacePromise; }); - it("delete", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("delete", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const deletionPromise = db1.getCollection(IBook).deleteMany({ filter: { id: 1 }, }); - deletionPromise.then((value) => { - expect(value.status).toBe('deleted: {"id":1}'); - }); + deletionPromise + .then((value) => { + expect(value.status).toBe('deleted: {"id":1}'); + }) + .catch((r) => console.log(r)); return deletionPromise; }); - it("deleteOne", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const collection = tigris.getDatabase().getCollection("books"); + it("deleteOne", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const collection = (await tigris.getDatabase()).getCollection("books"); const spyCollection = spy(collection); const expectedFilter = { id: 1 }; @@ -304,9 +308,9 @@ describe("rpc tests", () => { return deletePromise; }); - it("update", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("update", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const updatePromise = db1.getCollection("books").updateMany({ filter: { op: SelectorFilterOperator.EQ, @@ -328,9 +332,9 @@ describe("rpc tests", () => { return updatePromise; }); - it("updateOne", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const collection = tigris.getDatabase().getCollection("books"); + it("updateOne", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const collection = (await tigris.getDatabase()).getCollection("books"); const spyCollection = spy(collection); const expectedFilter = { id: 1 }; @@ -359,9 +363,9 @@ describe("rpc tests", () => { return updatePromise; }); - it("readOne", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("readOne", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const readOnePromise = db1.getCollection("books").findOne({ filter: { op: SelectorFilterOperator.EQ, @@ -380,9 +384,9 @@ describe("rpc tests", () => { return readOnePromise; }); - it("readOneRecordNotFound", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("readOneRecordNotFound", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const readOnePromise = db1.getCollection("books").findOne({ filter: { op: SelectorFilterOperator.EQ, @@ -397,9 +401,9 @@ describe("rpc tests", () => { return readOnePromise; }); - it("readOneWithLogicalFilter", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db1 = tigris.getDatabase(); + it("readOneWithLogicalFilter", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db1 = await tigris.getDatabase(); const readOnePromise: Promise = db1.getCollection("books").findOne({ filter: { op: LogicalOperator.AND, @@ -430,10 +434,10 @@ describe("rpc tests", () => { }); describe("findMany", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db = tigris.getDatabase(); + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); it("with filter using for await on cursor", async () => { + const db = await tigris.getDatabase(); const cursor = db.getCollection("books").findMany({ filter: { op: SelectorFilterOperator.EQ, @@ -451,7 +455,8 @@ describe("rpc tests", () => { expect(bookCounter).toBe(4); }); - it("finds all and retrieves results as array", () => { + it("finds all and retrieves results as array", async () => { + const db = await tigris.getDatabase(); const cursor = db.getCollection("books").findMany(); const booksPromise = cursor.toArray(); @@ -460,6 +465,7 @@ describe("rpc tests", () => { }); it("finds all and streams through results", async () => { + const db = await tigris.getDatabase(); const cursor = db.getCollection("books").findMany(); const booksIterator = cursor.stream(); @@ -472,6 +478,7 @@ describe("rpc tests", () => { }); it("throws an error", async () => { + const db = await tigris.getDatabase(); const cursor = db.getCollection("books").findMany({ filter: { op: SelectorFilterOperator.EQ, @@ -493,13 +500,13 @@ describe("rpc tests", () => { }); describe("search", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db = tigris.getDatabase(); + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); describe("with page number", () => { const pageNumber = 2; - it("returns a promise", () => { + it("returns a promise", async () => { + const db = await tigris.getDatabase(); const query: SearchQuery = { q: "philosophy", facets: { @@ -521,6 +528,7 @@ describe("rpc tests", () => { describe("without explicit page number", () => { it("returns an iterator", async () => { + const db = await tigris.getDatabase(); const query: SearchQuery = { q: "philosophy", facets: { @@ -543,9 +551,9 @@ describe("rpc tests", () => { }); }); - it("beginTx", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db3 = tigris.getDatabase(); + it("beginTx", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db3 = await tigris.getDatabase(); const beginTxPromise = db3.beginTransaction(); beginTxPromise.then((value) => { expect(value.id).toBe("id-test"); @@ -554,37 +562,37 @@ describe("rpc tests", () => { return beginTxPromise; }); - it("commitTx", (done) => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db3 = tigris.getDatabase(); + it("commitTx", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db3 = await tigris.getDatabase(); const beginTxPromise = db3.beginTransaction(); beginTxPromise.then((session) => { const commitTxResponse = session.commit(); commitTxResponse.then((value) => { expect(value.status).toBe("committed-test"); - done(); }); + return beginTxPromise; }); }); - it("rollbackTx", (done) => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db3 = tigris.getDatabase(); + it("rollbackTx", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db3 = await tigris.getDatabase(); const beginTxPromise = db3.beginTransaction(); beginTxPromise.then((session) => { const rollbackTransactionResponsePromise = session.rollback(); rollbackTransactionResponsePromise.then((value) => { expect(value.status).toBe("rollback-test"); - done(); }); }); + return beginTxPromise; }); - it("transact", (done) => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "test-tx" }); - const txDB = tigris.getDatabase(); + it("transact", async () => { + const tigris = new Tigris({ projectName: "test-tx", ...testConfig }); + const txDB = await tigris.getDatabase(); const books = txDB.getCollection("books"); - txDB.transact((tx) => { + return txDB.transact((tx) => { books .insertOne( { @@ -640,16 +648,16 @@ describe("rpc tests", () => { }, tx ) - .then(() => done()); + .then(); }); }); }); }); }); - it("createOrUpdateCollections", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); - const db3 = tigris.getDatabase(); + it("createOrUpdateCollections", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); + const db3 = await tigris.getDatabase(); const bookSchema: TigrisSchema = { id: { type: TigrisDataTypes.INT64, @@ -676,8 +684,8 @@ describe("rpc tests", () => { }); }); - it("serverMetadata", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); + it("serverMetadata", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); const serverMetadataPromise = tigris.getServerMetadata(); serverMetadataPromise.then((value) => { expect(value.serverVersion).toBe("1.0.0-test-service"); @@ -685,8 +693,8 @@ describe("rpc tests", () => { return serverMetadataPromise; }); - it("createCache", () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); + it("createCache", async () => { + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); const cacheC1Promise = tigris.createCacheIfNotExists("c1"); cacheC1Promise.then((value) => { expect(value.getCacheName()).toBe("c1"); @@ -695,7 +703,7 @@ describe("rpc tests", () => { }); it("listCaches", async () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); for (let i = 0; i < 5; i++) { await tigris.createCacheIfNotExists("c" + i); } @@ -716,7 +724,7 @@ describe("rpc tests", () => { }); it("deleteCache", async () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); for (let i = 0; i < 5; i++) { await tigris.createCacheIfNotExists("c" + i); } @@ -747,7 +755,7 @@ describe("rpc tests", () => { }); it("cacheCrud", async () => { - const tigris = new Tigris({ serverUrl: "localhost:" + SERVER_PORT, projectName: "db3" }); + const tigris = new Tigris({ ...testConfig, projectName: "db3" }); const c1 = await tigris.createCacheIfNotExists("c1"); await c1.set("k1", "val1"); @@ -774,6 +782,7 @@ describe("rpc tests", () => { await c1.del("k1"); let errored = false; + try { await c1.get("k1"); } catch (error) { @@ -803,6 +812,148 @@ describe("rpc tests", () => { console.log(error); } }); + + describe("DB branching", () => { + const tigris = new Tigris(testConfig); + + it("creates a new branch", async () => { + expect.hasAssertions(); + const db = await tigris.getDatabase(); + const createResp = db.createBranch("staging"); + + return createResp.then((r) => expect(r.status).toBe("created")); + }); + + it("fails to create existing branch", async () => { + expect.assertions(2); + const db = await tigris.getDatabase(); + const createResp = db.createBranch(Branch.Existing); + createResp.catch((r) => { + expect((r as ServiceError).code).toEqual(Status.ALREADY_EXISTS); + }); + + return expect(createResp).rejects.toBeDefined(); + }); + + it("deletes a branch successfully", async () => { + expect.hasAssertions(); + const db = await tigris.getDatabase(); + const deleteResp = db.deleteBranch("staging"); + + return deleteResp.then((r) => expect(r.status).toBe("deleted")); + }); + + it("fails to delete a branch if not existing already", async () => { + expect.assertions(2); + const db = await tigris.getDatabase(); + const deleteResp = db.deleteBranch(Branch.NotFound); + deleteResp.catch((r) => { + expect((r as ServiceError).code).toEqual(Status.NOT_FOUND); + }); + + return expect(deleteResp).rejects.toBeDefined(); + }); + }); + + describe("initializeDB() tests", () => { + let mockedUtil; + let config: TigrisClientConfig = { + serverUrl: testConfig.serverUrl, + projectName: testConfig.projectName, + }; + beforeEach(() => { + mockedUtil = spy(Utility); + }); + + afterEach(() => { + reset(mockedUtil); + }); + + it("uses 'default branch' when no branch name given", async () => { + expect(config["branch"]).toBeUndefined(); + when(mockedUtil.branchNameFromEnv(anything())).thenReturn(undefined); + const tigris = new Tigris(config); + const dbPromise = tigris.getDatabase(); + const db = await dbPromise; + expect(db.branch).toBe(""); + return dbPromise; + }); + + it("accepts empty string for branch name", async () => { + when(mockedUtil.branchNameFromEnv(anything())).thenReturn({ + name: "", + dynamicCreation: false, + }); + const tigris = new Tigris({ ...config, branch: "" }); + const dbPromise = tigris.getDatabase(); + const db = await dbPromise; + expect(db.branch).toBe(""); + return dbPromise; + }); + + it("skips creating branch for existing", async () => { + when(mockedUtil.branchNameFromEnv(anything())).thenReturn({ + name: "staging", + dynamicCreation: true, + }); + const tigris = new Tigris(config); + const dbPromise = tigris.getDatabase(); + const db = await dbPromise; + expect(db.branch).toBe("staging"); + return dbPromise; + }); + + it("creates templated branch if not exist", async () => { + when(mockedUtil.branchNameFromEnv(anything())).thenReturn({ + name: "fork_feature_1", + dynamicCreation: true, + }); + const tigris = new Tigris(config); + const dbPromise = tigris.getDatabase(); + const db = await dbPromise; + expect(db.branch).toBe("fork_feature_1"); + return dbPromise; + }); + + it("throws error if non-templated branch does not exist", async () => { + when(mockedUtil.branchNameFromEnv(anything())).thenReturn({ + name: "fork_feature_1", + dynamicCreation: false, + }); + const tigris = new Tigris(config); + const dbPromise = tigris.getDatabase(); + return await expect(dbPromise).rejects.toThrow(DatabaseBranchError); + }); + + it("fails to create branch if project does not exist", async () => { + when(mockedUtil.branchNameFromEnv(anything())).thenReturn({ + name: Branch.NotFound, + dynamicCreation: true, + }); + const tigris = new Tigris(config); + const dbPromise = tigris.getDatabase(); + try { + await dbPromise; + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as ServiceError).code).toEqual(Status.NOT_FOUND); + } + + return await expect(dbPromise).rejects.toThrow(Error); + }); + + it("initializer succeeds if branch already exists", async () => { + when(mockedUtil.branchNameFromEnv(anything())).thenReturn({ + name: Branch.Existing, + dynamicCreation: true, + }); + const tigris = new Tigris(config); + const dbPromise = tigris.getDatabase(); + const db = await dbPromise; + expect(db.branch).toBe(Branch.Existing); + return dbPromise; + }); + }); }); @TigrisCollection("books") diff --git a/src/__tests__/tigris.utility.spec.ts b/src/__tests__/tigris.utility.spec.ts index 8039315..be80808 100644 --- a/src/__tests__/tigris.utility.spec.ts +++ b/src/__tests__/tigris.utility.spec.ts @@ -80,18 +80,25 @@ describe("utility tests", () => { describe("createProtoSearchRequest", () => { const dbName = "my_test_db"; + const branch = "my_test_branch"; const collectionName = "my_test_collection"; it("populates projectName and collection name", () => { const emptyRequest = { q: "" }; - const generated = Utility.createProtoSearchRequest(dbName, collectionName, emptyRequest); + const generated = Utility.createProtoSearchRequest( + dbName, + branch, + collectionName, + emptyRequest + ); expect(generated.getProject()).toBe(dbName); + expect(generated.getBranch()).toBe(branch); expect(generated.getCollection()).toBe(collectionName); }); it("creates default match all query string", () => { const request = { q: undefined }; - const generated = Utility.createProtoSearchRequest(dbName, collectionName, request); + const generated = Utility.createProtoSearchRequest(dbName, branch, collectionName, request); expect(generated.getQ()).toBe(MATCH_ALL_QUERY_STRING); }); @@ -102,10 +109,83 @@ describe("utility tests", () => { }, }; const emptyRequest = { q: "", options: options }; - const generated = Utility.createProtoSearchRequest(dbName, collectionName, emptyRequest); + const generated = Utility.createProtoSearchRequest( + dbName, + branch, + collectionName, + emptyRequest + ); expect(generated.getPage()).toBe(0); expect(generated.getPageSize()).toBe(0); expect(generated.getCollation().getCase()).toBe("ci"); }); + + const nerfingTestCases = [ + ["main/fork", "main_fork"], + ["main-fork", "main_fork"], + ["main?fork", "main_fork"], + ["sTaging21", "sTaging21"], + ["hotfix/jira-23$4", "hotfix_jira_23_4"], + ["", ""], + ["release", "release"], + ["zero ops", "zero_ops"], + ["under_score", "under_score"], + ]; + + test.each(nerfingTestCases)("nerfs the name - '%s'", (original, nerfed) => { + expect(Utility.nerfGitBranchName(original)).toBe(nerfed); + }); + + describe("get branch name from environment", () => { + const OLD_ENV = Object.assign({}, process.env); + + beforeEach(() => { + jest.resetModules(); + }); + + afterEach(() => { + process.env = OLD_ENV; + }); + + it.each([ + [ + "preview_${GIT_BRANCH}", + "GIT_BRANCH", + "feature_1", + { name: "preview_feature_1", dynamicCreation: true }, + ], + ["staging", undefined, undefined, { name: "staging", dynamicCreation: false }], + ["integration_${MY_VAR}_auto", undefined, undefined, undefined], + ["integration_${MY_VAR}_auto", "NOT_SET", "feature_2", undefined], + [ + "${MY_GIT_BRANCH}", + "MY_GIT_BRANCH", + "jira/1234", + { name: "jira_1234", dynamicCreation: true }, + ], + [ + "${MY_GIT_BRANCH", + "MY_GIT_BRANCH", + "jira/1234", + { name: "${MY_GIT_BRANCH", dynamicCreation: false }, + ], + [undefined, undefined, undefined, undefined], + ])("envVar - '%s'", (branchEnvValue, templateEnvKey, templateEnvValue, expected) => { + process.env["TIGRIS_DB_BRANCH"] = branchEnvValue; + if (templateEnvKey) { + process.env[templateEnvKey] = templateEnvValue; + } + expect(Utility.branchNameFromEnv()).toEqual(expected); + }); + + it.each([ + ["any_given_branch", "any_given_branch"], + ["", ""], + ])("given branch - '%s'", (givenBranch, expected) => { + const actual = Utility.branchNameFromEnv(givenBranch); + expect(actual.name).toBe(expected); + expect(actual.dynamicCreation).toBeFalsy(); + }); + }); }); }); diff --git a/src/__tests__/utils/env-loader.spec.ts b/src/__tests__/utils/env-loader.spec.ts new file mode 100644 index 0000000..c8f3788 --- /dev/null +++ b/src/__tests__/utils/env-loader.spec.ts @@ -0,0 +1,17 @@ +import { initializeEnvironment } from "../../utils/env-loader"; + +describe("configLoader", () => { + const OLD_ENV = Object.assign({}, process.env); + + beforeEach(() => { + jest.resetModules(); + }); + + afterEach(() => { + process.env = OLD_ENV; + }); + + it("no side effect if files not found", () => { + initializeEnvironment(); + }); +}); diff --git a/src/collection.ts b/src/collection.ts index cb4186e..435cd99 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -44,17 +44,20 @@ interface ICollection { export class Collection implements ICollection { readonly collectionName: string; readonly db: string; + readonly branch: string; readonly grpcClient: TigrisClient; readonly config: TigrisClientConfig; constructor( collectionName: string, db: string, + branch: string, grpcClient: TigrisClient, config: TigrisClientConfig ) { this.collectionName = collectionName; this.db = db; + this.branch = branch; this.grpcClient = grpcClient; this.config = config; } @@ -74,6 +77,7 @@ export class Collection implements ICollection { const protoRequest = new ProtoInsertRequest() .setProject(this.db) + .setBranch(this.branch) .setCollection(this.collectionName) .setDocumentsList(docsArray); @@ -124,6 +128,7 @@ export class Collection implements ICollection { ); const protoRequest = new ProtoReplaceRequest() .setProject(this.db) + .setBranch(this.branch) .setCollection(this.collectionName) .setDocumentsList(docsArray); @@ -202,6 +207,7 @@ export class Collection implements ICollection { return new Promise((resolve, reject) => { const updateRequest = new ProtoUpdateRequest() .setProject(this.db) + .setBranch(this.branch) .setCollection(this.collectionName) .setFilter(Utility.stringToUint8Array(Utility.filterToString(query.filter))) .setFields(Utility.stringToUint8Array(Utility.updateFieldsString(query.fields))); @@ -330,6 +336,7 @@ export class Collection implements ICollection { } const deleteRequest = new ProtoDeleteRequest() .setProject(this.db) + .setBranch(this.branch) .setCollection(this.collectionName) .setFilter(Utility.stringToUint8Array(Utility.filterToString(query.filter))); @@ -501,6 +508,7 @@ export class Collection implements ICollection { } const readRequest = new ProtoReadRequest() .setProject(this.db) + .setBranch(this.branch) .setCollection(this.collectionName) .setFilter(Utility.stringToUint8Array(Utility.filterToString(query.filter))); @@ -665,6 +673,7 @@ export class Collection implements ICollection { search(query: SearchQuery, page?: number): SearchIterator | Promise> { const searchRequest = Utility.createProtoSearchRequest( this.db, + this.branch, this.collectionName, query, page diff --git a/src/db.ts b/src/db.ts index 7ead354..29d4409 100644 --- a/src/db.ts +++ b/src/db.ts @@ -5,8 +5,10 @@ import { CollectionMetadata, CollectionOptions, CommitTransactionResponse, + CreateBranchResponse, DatabaseDescription, DatabaseMetadata, + DeleteBranchResponse, DropCollectionResponse, TigrisCollectionType, TigrisSchema, @@ -17,7 +19,9 @@ import { BeginTransactionRequest as ProtoBeginTransactionRequest, BeginTransactionResponse, CollectionOptions as ProtoCollectionOptions, + CreateBranchRequest as ProtoCreateBranchRequest, CreateOrUpdateCollectionRequest as ProtoCreateOrUpdateCollectionRequest, + DeleteBranchRequest as ProtoDeleteBranchRequest, DescribeDatabaseRequest as ProtoDescribeDatabaseRequest, DropCollectionRequest as ProtoDropCollectionRequest, ListCollectionsRequest as ProtoListCollectionsRequest, @@ -31,28 +35,117 @@ import { DecoratedSchemaProcessor } from "./schema/decorated-schema-processor"; import { Log } from "./utils/logger"; import { DecoratorMetaStorage } from "./decorators/metadata/decorator-meta-storage"; import { getDecoratorMetaStorage } from "./globals"; -import { CollectionNotFoundError } from "./error"; +import { CollectionNotFoundError, DatabaseBranchError } from "./error"; +import { Status } from "@grpc/grpc-js/build/src/constants"; -/** - * Tigris Database - */ const SetCookie = "Set-Cookie"; const Cookie = "Cookie"; const BeginTransactionMethodName = "/tigrisdata.v1.Tigris/BeginTransaction"; +/** + * A branch name could be dynamically generated from environment variables. + * + * @example Simple name "my_database_branch" would translate to: + * ``` + * { + * name: "my_database_branch", + * isTemplated: false + * } + * ``` + * @example A dynamically generated branch name "my_db_${GIT_BRANCH}" would translate to: + * ``` + * export GIT_BRANCH=feature_1 + * { + * name: "my_db_feature_1", + * isTemplated: true + * } + * ``` + */ +export type TemplatedBranchName = { name: string; dynamicCreation: boolean }; +const NoBranch: TemplatedBranchName = { name: "", dynamicCreation: false }; + +/** + * Tigris Database class to manage database branches, collections and execute + * transactions. + */ export class DB { private readonly _db: string; + private _branchVar: TemplatedBranchName; private readonly grpcClient: TigrisClient; private readonly config: TigrisClientConfig; private readonly schemaProcessor: DecoratedSchemaProcessor; private readonly _metadataStorage: DecoratorMetaStorage; + private readonly _ready: Promise; + /** + * Create an instance of Tigris Database class. + * + * @example Recommended way to create instance using {@link TigrisClient.getDatabase} + * ``` + * const client = new TigrisClient(); + * const db = await client.getDatabase(); + * ``` + * + * @remarks + * Highly recommend to use {@link TigrisClient.getDatabase} to create instance of this class and + * not attempt to instantiate this class directly, it can have potential side effects + * if database branch is not initialized properly. + * + * @privateRemarks + * Object of this class depends on readiness state for proper initialization of database + * and branch. To ensure object is ready to use: + * ``` + * const instance = new DB(name, client, config); + * const db: DB = await instance.ready; + * + * db.describe(); + * ``` + */ constructor(db: string, grpcClient: TigrisClient, config: TigrisClientConfig) { this._db = db; this.grpcClient = grpcClient; this.config = config; this.schemaProcessor = DecoratedSchemaProcessor.Instance; this._metadataStorage = getDecoratorMetaStorage(); + // TODO: Should we just default to `main` or empty arg or throw an exception here? + this._branchVar = Utility.branchNameFromEnv(config.branch) ?? NoBranch; + this._ready = this.initializeDB(); + } + + /** + * Initializes a database branch and returns DB object. A DB shouldn't be used + * until it is initialized. + * + * Calls {@link describe()} to assert that the branch in use already exists. If not, and the + * branch name needs to be generated dynamically (ex - `preview_${GIT_BRANCH}`) then try to + * create that branch. + * + * @throws Error if branch doesn't exist and/or cannot be created + * @private + */ + private async initializeDB(): Promise { + if (this._branchVar.name === NoBranch.name) { + return this; + } + const description = await this.describe(); + const branchExists = description.branches.includes(this.branch); + + if (!branchExists) { + if (this._branchVar.dynamicCreation) { + try { + await this.createBranch(this.branch); + } catch (error) { + if ((error as ServiceError).code !== Status.ALREADY_EXISTS) { + throw error; + } + } + Log.event(`Created database branch: ${this.branch}`); + } else { + throw new DatabaseBranchError(this.branch); + } + } + Log.info(`Using database branch: '${this.branch}'`); + return this; } /** @@ -125,7 +218,7 @@ export class DB { return this.createOrUpdate( collectionName, schema, - () => new Collection(collectionName, this._db, this.grpcClient, this.config) + () => new Collection(collectionName, this._db, this.branch, this.grpcClient, this.config) ); } @@ -138,6 +231,7 @@ export class DB { const rawJSONSchema: string = Utility._toJSONSchema(name, schema); const createOrUpdateCollectionRequest = new ProtoCreateOrUpdateCollectionRequest() .setProject(this._db) + .setBranch(this.branch) .setCollection(name) .setOnlyCreate(false) .setSchema(Utility.stringToUint8Array(rawJSONSchema)); @@ -159,7 +253,7 @@ export class DB { public listCollections(options?: CollectionOptions): Promise> { return new Promise>((resolve, reject) => { - const request = new ProtoListCollectionsRequest().setProject(this.db); + const request = new ProtoListCollectionsRequest().setProject(this.db).setBranch(this.branch); if (typeof options !== "undefined") { return request.setOptions(new ProtoCollectionOptions()); } @@ -198,7 +292,10 @@ export class DB { const collectionName = this.resolveNameFromCollectionClass(nameOrClass); return new Promise((resolve, reject) => { this.grpcClient.dropCollection( - new ProtoDropCollectionRequest().setProject(this.db).setCollection(collectionName), + new ProtoDropCollectionRequest() + .setProject(this.db) + .setBranch(this.branch) + .setCollection(collectionName), (error, response) => { if (error) { reject(error); @@ -226,7 +323,7 @@ export class DB { public describe(): Promise { return new Promise((resolve, reject) => { this.grpcClient.describeDatabase( - new ProtoDescribeDatabaseRequest().setProject(this.db), + new ProtoDescribeDatabaseRequest().setProject(this.db).setBranch(this.branch), (error, response) => { if (error) { reject(error); @@ -241,7 +338,13 @@ export class DB { ) ); } - resolve(new DatabaseDescription(new DatabaseMetadata(), collectionsDescription)); + resolve( + new DatabaseDescription( + new DatabaseMetadata(), + collectionsDescription, + response.getBranchesList() + ) + ); } } ); @@ -266,7 +369,7 @@ export class DB { public getCollection(nameOrClass: T | string): Collection { const collectionName = this.resolveNameFromCollectionClass(nameOrClass); - return new Collection(collectionName, this.db, this.grpcClient, this.config); + return new Collection(collectionName, this.db, this.branch, this.grpcClient, this.config); } private resolveNameFromCollectionClass(nameOrClass: TigrisCollectionType | string) { @@ -312,7 +415,9 @@ export class DB { // eslint-disable-next-line @typescript-eslint/no-unused-vars public beginTransaction(_options?: TransactionOptions): Promise { return new Promise((resolve, reject) => { - const beginTxRequest = new ProtoBeginTransactionRequest().setProject(this._db); + const beginTxRequest = new ProtoBeginTransactionRequest() + .setProject(this._db) + .setBranch(this.branch); const cookie: Metadata = new Metadata(); const call = this.grpcClient.makeUnaryRequest( BeginTransactionMethodName, @@ -331,6 +436,7 @@ export class DB { response.getTxCtx().getOrigin(), this.grpcClient, this.db, + this.branch, cookie ) ); @@ -345,7 +451,41 @@ export class DB { }); } + public createBranch(name: string): Promise { + return new Promise((resolve, reject) => { + const req = new ProtoCreateBranchRequest().setProject(this.db).setBranch(name); + this.grpcClient.createBranch(req, (error, response) => { + if (error) { + reject(error); + return; + } + resolve(CreateBranchResponse.from(response)); + }); + }); + } + + public deleteBranch(name: string): Promise { + return new Promise((resolve, reject) => { + const req = new ProtoDeleteBranchRequest().setProject(this.db).setBranch(name); + this.grpcClient.deleteBranch(req, (error, response) => { + if (error) { + reject(error); + return; + } + resolve(DeleteBranchResponse.from(response)); + }); + }); + } + get db(): string { return this._db; } + + get branch(): string { + return this._branchVar.name; + } + + get ready(): Promise { + return this._ready; + } } diff --git a/src/error.ts b/src/error.ts index 2fe419e..296f9c9 100644 --- a/src/error.ts +++ b/src/error.ts @@ -93,3 +93,18 @@ export class CollectionNotFoundError extends TigrisError { return "CollectionNotFoundError"; } } + +export class DatabaseBranchError extends TigrisError { + constructor(name?: string) { + const errMsg = name + ? `Database branch ${name} does not exist.` + : "Database branch name environment variable is required. " + + "Run 'export TIGRIS_DB_BRANCH=YOUR_BRANCH_NAME' or include it in your environment file."; + + super(errMsg); + } + + override get name(): string { + return "DatabaseBranchError"; + } +} diff --git a/src/session.ts b/src/session.ts index 7965034..9451e04 100644 --- a/src/session.ts +++ b/src/session.ts @@ -12,6 +12,7 @@ export class Session { private readonly _origin: string; private readonly grpcClient: TigrisClient; private readonly db: string; + private readonly branch: string; private readonly _additionalMetadata: Metadata; constructor( @@ -19,12 +20,14 @@ export class Session { origin: string, grpcClient: TigrisClient, db: string, + branch: string, additionalMetadata: Metadata ) { this._id = id; this._origin = origin; this.grpcClient = grpcClient; this.db = db; + this.branch = branch; this._additionalMetadata = additionalMetadata; } @@ -42,7 +45,9 @@ export class Session { public commit(): Promise { return new Promise((resolve, reject) => { - const request = new ProtoCommitTransactionRequest().setProject(this.db); + const request = new ProtoCommitTransactionRequest() + .setProject(this.db) + .setBranch(this.branch); this.grpcClient.commitTransaction(request, Utility.txToMetadata(this), (error, response) => { if (error) { reject(error); @@ -55,7 +60,9 @@ export class Session { public rollback(): Promise { return new Promise((resolve, reject) => { - const request = new ProtoRollbackTransactionRequest().setProject(this.db); + const request = new ProtoRollbackTransactionRequest() + .setProject(this.db) + .setBranch(this.branch); this.grpcClient.rollbackTransaction( request, Utility.txToMetadata(this), diff --git a/src/tigris.ts b/src/tigris.ts index 91fab93..0e43a4d 100644 --- a/src/tigris.ts +++ b/src/tigris.ts @@ -6,7 +6,6 @@ import { ChannelCredentials, Metadata } from "@grpc/grpc-js"; import { GetInfoRequest as ProtoGetInfoRequest } from "./proto/server/v1/observability_pb"; import { HealthCheckInput as ProtoHealthCheckInput } from "./proto/server/v1/health_pb"; -import * as dotenv from "dotenv"; import { DeleteCacheResponse, ListCachesResponse, @@ -33,6 +32,7 @@ import { DeleteCacheRequest as ProtoDeleteCacheRequest } from "./proto/server/v1 import { ListCachesRequest as ProtoListCachesRequest } from "./proto/server/v1/cache_pb"; import { Status } from "@grpc/grpc-js/build/src/constants"; +import { initializeEnvironment } from "./utils/env-loader"; const AuthorizationHeaderName = "authorization"; const AuthorizationBearer = "Bearer "; @@ -64,6 +64,11 @@ export interface TigrisClientConfig { * Controls the ping interval, if not specified defaults to 300_000ms (i.e. 5 min) */ pingIntervalMs?: number; + + /** + * Database branch name + */ + branch?: string; } class TokenSupplier { @@ -149,7 +154,7 @@ export class Tigris { * @param {TigrisClientConfig} config configuration */ constructor(config?: TigrisClientConfig) { - dotenv.config(); + initializeEnvironment(); if (typeof config === "undefined") { config = {}; } @@ -258,8 +263,9 @@ export class Tigris { Log.info(`Using Tigris at: ${config.serverUrl}`); } - public getDatabase(): DB { - return new DB(this._config.projectName, this.grpcClient, this._config); + public getDatabase(): Promise { + const db = new DB(this._config.projectName, this.grpcClient, this._config); + return db.ready; } /** diff --git a/src/types.ts b/src/types.ts index f1cc486..a2551ea 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,8 @@ import { Collation } from "./search/types"; +import { + CreateBranchResponse as ProtoCreateBranchResponse, + DeleteBranchResponse as ProtoDeleteBranchResponse, +} from "./proto/server/v1/api_pb"; export class DatabaseInfo { private readonly _name: string; @@ -44,22 +48,49 @@ export class DatabaseOptions {} export class CollectionOptions {} -export class DropDatabaseResponse { +export class TigrisResponse { private readonly _status: string; - private readonly _message: string; - constructor(status: string, message: string) { + constructor(status: string) { this._status = status; - this._message = message; } get status(): string { return this._status; } +} + +export class CreateBranchResponse extends TigrisResponse { + private readonly _message: string; + + constructor(status: string, message: string) { + super(status); + this._message = message; + } get message(): string { return this._message; } + + static from(response: ProtoCreateBranchResponse): CreateBranchResponse { + return new this(response.getStatus(), response.getMessage()); + } +} + +export class DeleteBranchResponse extends TigrisResponse { + private readonly _message: string; + constructor(status: string, message: string) { + super(status); + this._message = message; + } + + get message(): string { + return this._message; + } + + static from(response: ProtoDeleteBranchResponse): DeleteBranchResponse { + return new this(response.getStatus(), response.getMessage()); + } } export class DropCollectionResponse { @@ -82,20 +113,30 @@ export class DropCollectionResponse { export class DatabaseDescription { private readonly _metadata: DatabaseMetadata; - private readonly _collectionsDescription: Array; - - constructor(metadata: DatabaseMetadata, collectionsDescription: Array) { + private readonly _collectionsDescription: ReadonlyArray; + private readonly _branches: ReadonlyArray; + + constructor( + metadata: DatabaseMetadata, + collectionsDescription: Array, + branches: Array + ) { this._metadata = metadata; this._collectionsDescription = collectionsDescription; + this._branches = branches; } get metadata(): DatabaseMetadata { return this._metadata; } - get collectionsDescription(): Array { + get collectionsDescription(): ReadonlyArray { return this._collectionsDescription; } + + get branches(): ReadonlyArray { + return this._branches; + } } export class CollectionDescription { @@ -122,18 +163,6 @@ export class CollectionDescription { } } -export class TigrisResponse { - private readonly _status: string; - - constructor(status: string) { - this._status = status; - } - - get status(): string { - return this._status; - } -} - export class DMLMetadata { private readonly _createdAt: Date; private readonly _updatedAt: Date; diff --git a/src/utility.ts b/src/utility.ts index 586075a..a0aa1b8 100644 --- a/src/utility.ts +++ b/src/utility.ts @@ -37,6 +37,7 @@ import { UpdateRequestOptions as ProtoUpdateRequestOptions, } from "./proto/server/v1/api_pb"; import { TigrisClientConfig } from "./tigris"; +import { TemplatedBranchName } from "./db"; export const Utility = { stringToUint8Array(input: string): Uint8Array { @@ -47,6 +48,39 @@ export const Utility = { return new TextDecoder().decode(input); }, + /** @see tests for usage */ + branchNameFromEnv(given?: string): TemplatedBranchName { + const maybeBranchName = typeof given !== "undefined" ? given : process.env.TIGRIS_DB_BRANCH; + if (typeof maybeBranchName === "undefined") { + return undefined; + } + const isTemplate = Utility.getTemplatedVar(maybeBranchName); + if (isTemplate) { + return isTemplate.extracted in process.env + ? { + name: maybeBranchName.replace( + isTemplate.matched, + this.nerfGitBranchName(process.env[isTemplate.extracted]) + ), + dynamicCreation: true, + } + : undefined; + } else { + return { name: maybeBranchName, dynamicCreation: false }; + } + }, + + /** @see {@link branchNameFromEnv} tests for usage */ + getTemplatedVar(input: string): { matched: string; extracted: string } { + const output = input.match(/\${(.*?)}/); + return output ? { matched: output[0], extracted: output[1] } : undefined; + }, + + /** @see tests for usage */ + nerfGitBranchName(original: string) { + return original.replace(/[^\d\n.A-Za-z]/g, "_"); + }, + filterToString(filter: Filter): string { if ( Object.prototype.hasOwnProperty.call(filter, "op") && @@ -505,12 +539,14 @@ export const Utility = { createProtoSearchRequest( dbName: string, + branch: string, collectionName: string, query: SearchQuery, page?: number ): ProtoSearchRequest { const searchRequest = new ProtoSearchRequest() .setProject(dbName) + .setBranch(branch) .setCollection(collectionName) .setQ(query.q ?? MATCH_ALL_QUERY_STRING); diff --git a/src/utils/env-loader.ts b/src/utils/env-loader.ts new file mode 100644 index 0000000..f814da2 --- /dev/null +++ b/src/utils/env-loader.ts @@ -0,0 +1,67 @@ +import appRootPath from "app-root-path"; +import * as dotenv from "dotenv"; +import path from "node:path"; +import fs from "node:fs"; +import { Log } from "./logger"; + +/** + * Uses dotenv() to initialize `.env` config files based on the **NODE_ENV** + * environment variable in order -: + * 1. `.env.${NODE_ENV}.local` + * 2. `.env.local` + * 3. `.env.${NODE_ENV}` + * 4. `.env` + * + * @example If `NODE_ENV = production` + * ``` + * export NODE_ENV=production + * + * // will load following 4 config files in order: + * .env.production.local + * .env.local + * .env.production + * .env + * ``` + */ +export function initializeEnvironment() { + const envFiles = getEnvFiles(appRootPath.toString()); + for (const f of envFiles) { + dotenv.config({ path: f }); + } +} + +function getEnvFiles(dir: string) { + const nodeEnv = process.env.NODE_ENV; + const dotEnvFiles: Array = []; + switch (nodeEnv) { + case "test": + dotEnvFiles.push(`.env.${nodeEnv}.local`); + break; + case "development": + case "production": + dotEnvFiles.push(`.env.${nodeEnv}.local`, ".env.local"); + break; + } + dotEnvFiles.push(`.env.${nodeEnv}`, ".env"); + + const envFilePaths = []; + for (const envFile of dotEnvFiles) { + const envFilePath = path.join(dir, envFile); + try { + const stats = fs.statSync(envFilePath); + + // make sure to only attempt to read files + if (!stats.isFile()) { + continue; + } + + envFilePaths.push(envFilePath); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + if (error.code !== "ENOENT") { + Log.error(`Failed to read env from '${envFile}'`, error.message); + } + } + } + return envFilePaths; +}