Skip to content

Commit

Permalink
Merge branch 'main' into kc/nested-filtering-and-sorting
Browse files Browse the repository at this point in the history
  • Loading branch information
codingkarthik committed Sep 18, 2024
2 parents 5f8e827 + a803b83 commit 5478d2a
Show file tree
Hide file tree
Showing 8 changed files with 343 additions and 187 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ supportedEnvironmentVariables:
description: Name of the Azure Cosmos DB for NoSQL DB
- name: AZURE_COSMOS_ENDPOINT
description: Endpoint of the Azure Cosmos DB for NoSQL DB
- name: AZURE_COSMOS_MANAGED_CLIENT_ID
description: Managed client ID of the Azure Cosmos DB for NoSQL
- name: AZURE_COSMOS_NO_OF_ROWS_TO_FETCH
description: Maximum number of rows to fetch per container to infer the schema of the container.
default: "100"
commands:
update: docker run --rm -e AZURE_COSMOS_KEY="$AZURE_COSMOS_KEY" -e AZURE_COSMOS_DB_NAME="$AZURE_COSMOS_DB_NAME" -e AZURE_COSMOS_ENDPOINT="$AZURE_COSMOS_ENDPOINT" -v "$HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH":/etc/connector ghcr.io/hasura/ndc-azure-cosmos:v0.1.6 update
update: docker run --rm -e AZURE_COSMOS_KEY="$AZURE_COSMOS_KEY" -e AZURE_COSMOS_DB_NAME="$AZURE_COSMOS_DB_NAME" -e AZURE_COSMOS_ENDPOINT="$AZURE_COSMOS_ENDPOINT" -e AZURE_COSMOS_MANAGED_CLIENT_ID="$AZURE_COSMOS_MANAGED_CLIENT_ID" -v "$HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH":/etc/connector ghcr.io/hasura/ndc-azure-cosmos:v0.1.6 update
280 changes: 148 additions & 132 deletions package-lock.json

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,11 @@
},
"dependencies": {
"@azure/cosmos": "^4.1.0",
"fs-extra": "^11.2.0",
"@hasura/ndc-sdk-typescript": "^6.1.0",
"@azure/identity": "^4.0.1",
"@hasura/ndc-sdk-typescript": "^6.1.0",
"@types/fs-extra": "^11.0.4",
"dotenv": "^16.4.5",
"fs-extra": "^11.2.0",
"quicktype-core": "^23.0.104",
"ts-node": "^10.9.2",
"typescript": "^5.5.4"
Expand Down
33 changes: 27 additions & 6 deletions src/cli/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import {
NamedObjectTypeDefinition,
ScalarTypeDefinitions,
getJSONScalarTypes,
} from "../connector/schema";
import {
BuiltInScalarTypeName,
ObjectTypeDefinitions,
TypeDefinition,
Expand All @@ -21,7 +19,12 @@ import {
jsonInputForTargetLanguage,
quicktype,
} from "quicktype-core";
import { runSQLQuery, constructCosmosDbClient } from "../connector/db/cosmosDb";
import {
runSQLQuery,
constructCosmosDbClient,
AzureCosmosAuthenticationConfig,
} from "../connector/db/cosmosDb";

import { exit } from "process";
import fs from "fs";
import { promisify } from "util";
Expand Down Expand Up @@ -285,6 +288,23 @@ async function getCollectionsSchema(
return schema;
}

type UpdateConfig = {
version: 2;
connection: {
endpoint: string;
databaseName: string;
authentication: AzureCosmosAuthenticationConfig;
};
schema: CollectionsSchema;
};

/**
* Generates the connector configuration for Azure Cosmos DB for NoSQL DB. This function fetches the schema
* of the containers present in the specified Azure Cosmos DB and writes the configuration to the specified
* `outputConfigDir`.
* @param {string} outputConfigDir - Output Directory where the config file will be written.
*/
export async function generateConnectorConfig(outputConfigDir: string) {
const rowsToFetch = process.env["AZURE_COSMOS_NO_OF_ROWS_TO_FETCH"] ?? "100";

Expand All @@ -294,14 +314,15 @@ export async function generateConnectorConfig(outputConfigDir: string) {
client.dbClient,
parseInt(rowsToFetch),
);
const cosmosKey = client.connectionDetails.key;
const connectionConfig = client.connectionDetails.connectionConfig;
const cosmosEndpoint = client.connectionDetails.endpoint;
const cosmosDbName = client.connectionDetails.databaseName;

const response: any = {
const response: UpdateConfig = {
version: 2,
connection: {
endpoint: cosmosEndpoint,
key: cosmosKey,
authentication: connectionConfig,
databaseName: cosmosDbName,
},
schema,
Expand Down
25 changes: 21 additions & 4 deletions src/connector/connector.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import * as sdk from "@hasura/ndc-sdk-typescript";
import { CollectionsSchema, getNdcSchemaResponse } from "./schema";
import { constructCosmosDbClient } from "./db/cosmosDb";
import {
AzureCosmosAuthenticationConfig,
getCosmosDbClient,
} from "./db/cosmosDb";
import { Database } from "@azure/cosmos";
import { executeQuery } from "./execution";
import { readFileSync } from "fs";
Expand All @@ -10,7 +13,7 @@ export type Configuration = ConnectorConfig;
export type ConnectorConfig = {
connection: {
endpoint: string;
key: string;
authentication: AzureCosmosAuthenticationConfig;
databaseName: string;
};
schema: CollectionsSchema;
Expand Down Expand Up @@ -40,11 +43,25 @@ export function createConnector(): sdk.Connector<Configuration, State> {
},

tryInitState: async function (
_: Configuration,
config: Configuration,
__: unknown,
): Promise<State> {
try {
const databaseClient = constructCosmosDbClient().dbClient;
const {
databaseName,
authentication: authenticationConfig,
endpoint,
} = config.connection;
console.log(
"Initializing the state of the connector",
authenticationConfig,
);
const databaseClient = getCosmosDbClient(
endpoint,
databaseName,
authenticationConfig,
);

return Promise.resolve({
databaseClient,
});
Expand Down
159 changes: 123 additions & 36 deletions src/connector/db/cosmosDb.ts
Original file line number Diff line number Diff line change
@@ -1,55 +1,142 @@
import { CosmosClient, Database, Container, SqlQuerySpec } from "@azure/cosmos"
import * as sdk from '@hasura/ndc-sdk-typescript'
import { CosmosClient, Database, Container, SqlQuerySpec } from "@azure/cosmos";
import { DefaultAzureCredential } from "@azure/identity";
import { throwError } from "../../utils";

export type ManagedIdentityConfig = {
type: "ManagedIdentity";
// Name of the ENV var where the key can be found
fromEnvVar: string;
};

export type CosmosKeyConfig = {
type: "Key";
// Name of the ENV var where the key can be found
fromEnvVar: string;
};

export type AzureCosmosAuthenticationConfig =
| CosmosKeyConfig
| ManagedIdentityConfig;

export type RawCosmosDbConfig = {
databaseName: string,
endpoint: string,
key: string
}
databaseName: string;
endpoint: string;
connectionConfig: AzureCosmosAuthenticationConfig;
};

/* Creates a new cosmos DB client with which the specified database can be queried. */
function getCosmosDbClient(rawDbConfig: RawCosmosDbConfig): Database {
const dbClient = new CosmosClient({
key: rawDbConfig.key,
endpoint: rawDbConfig.endpoint,
});
export function getCosmosDbClient(
endpoint: string,
databaseName: string,
connectionConfig: AzureCosmosAuthenticationConfig,
): Database {
let dbClient: CosmosClient;
switch (connectionConfig.type) {
case "Key":
const key =
getEnvVariable(connectionConfig.fromEnvVar, true) ??
throwError(
`Azure Cosmos Key not found in the env var "${connectionConfig.fromEnvVar}"`,
);
dbClient = new CosmosClient({
key,
endpoint,
});
break;
case "ManagedIdentity":
const managedIdentityClientId =
getEnvVariable(connectionConfig.fromEnvVar, true) ??
throwError(
`Azure Cosmos Key not found in the env var "${connectionConfig.fromEnvVar}"`,
);

return dbClient.database(rawDbConfig.databaseName);
let credentials = new DefaultAzureCredential({
managedIdentityClientId,
});
dbClient = new CosmosClient({
endpoint,
aadCredentials: credentials,
});
break;
}

return dbClient.database(databaseName);
}

function getEnvVariable(envVarName: string): string {
const envVariable = process.env[envVarName];
if (!envVariable) {
throw new Error(`${envVarName} environment variable is not defined.`);
function getEnvVariable(
envVarName: string,
isRequired?: undefined | boolean,
): string | null {
const envVariable = process.env[envVarName];
if (!envVariable) {
if (isRequired) {
throw new Error(`${envVarName} environment variable is not defined.`);
} else {
return null;
}
return envVariable;
}
return envVariable;
}

export function constructCosmosDbClient() {
const key = getEnvVariable("AZURE_COSMOS_KEY");
const endpoint = getEnvVariable("AZURE_COSMOS_ENDPOINT");
const databaseName = getEnvVariable("AZURE_COSMOS_DB_NAME");

const dbClient = getCosmosDbClient({
databaseName, endpoint, key
});
function getConnectionConfig(): AzureCosmosAuthenticationConfig | null {
const key = getEnvVariable("AZURE_COSMOS_KEY");
const managed_identity_client_id = getEnvVariable(
"AZURE_COSMOS_MANAGED_CLIENT_ID",
);

const connectionDetails = {
endpoint,
key,
databaseName
if (key === null && managed_identity_client_id === null) {
throw new Error(
`Either the AZURE_COSMOS_KEY or the AZURE_COSMOS_MANAGED_CLIENT_ID env var is expected`,
);
} else if (key && managed_identity_client_id) {
throw new Error(
`Both AZURE_COSMOS_KEY and the AZURE_COSMOS_MANAGED_CLIENT_ID cannot be set`,
);
} else {
if (key) {
return {
type: "Key",
fromEnvVar: "AZURE_COSMOS_KEY",
};
} else if (managed_identity_client_id) {
return {
type: "ManagedIdentity",
fromEnvVar: "AZURE_COSMOS_MANAGED_CLIENT_ID",
};
}
}
return null;
}

return {
connectionDetails,
dbClient
}
export function constructCosmosDbClient() {
const endpoint =
getEnvVariable("AZURE_COSMOS_ENDPOINT", true) ??
throwError("AZURE_COSMOS_ENDPOINT not found");
const databaseName =
getEnvVariable("AZURE_COSMOS_DB_NAME", true) ??
throwError("AZURE_COSMOS_DB_NAME not found");
const connectionConfig =
getConnectionConfig() ??
throwError("internal: could not get the connection config");

}
const dbClient = getCosmosDbClient(endpoint, databaseName, connectionConfig);

const connectionDetails = {
endpoint,
databaseName,
connectionConfig,
};

return {
connectionDetails,
dbClient,
};
}

/* Runs the `sqlQuerySpec` in the specified `container` */
export async function runSQLQuery<T>(sqlQuerySpec: SqlQuerySpec, container: Container): Promise<T[]> {
return (await container.items.query(sqlQuerySpec).fetchAll()).resources
export async function runSQLQuery<T>(
sqlQuerySpec: SqlQuerySpec,
container: Container,
): Promise<T[]> {
return (await container.items.query(sqlQuerySpec).fetchAll()).resources;
}
19 changes: 14 additions & 5 deletions src/connector/execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ function selectNestedField(
function selectField(field: sdk.Field, fieldPrefix: string): sql.SelectColumn {
switch (field.type) {
case "column":
const column: sql.Column = {
const column = {
name: field.column,
prefix: fieldPrefix,
};
if (field.fields) {
if (field.fields !== null && field.fields !== undefined) {
const [nestedFieldSelectCol, _] = selectNestedField(
field.fields,
column,
Expand Down Expand Up @@ -144,7 +144,7 @@ function getRequestedFieldsFromObject(
return requestedFields;
}

export function getBaseType(typeDefn: schema.TypeDefinition): string {
function getBaseType(typeDefn: schema.TypeDefinition): string {
switch (typeDefn.type) {
case "array":
return getBaseType(typeDefn.elementType);
Expand Down Expand Up @@ -268,8 +268,11 @@ function parseExpression(
// write a function getBinaryComparisonOperator to get the type of the `comparisonTarget` column
// if the `comparisonTarget` contains a nested field, then we need to get the type of the nested field

console.log("comparisonTarget: ", comparisonTarget);
const comparisonTargetType = sql.getScalarType(comparisonTarget);

console.log("comparisonTargetType: ", comparisonTargetType);

const scalarDbOperator = sql.getDbComparisonOperator(
comparisonTargetType,
expression.operator,
Expand Down Expand Up @@ -341,7 +344,10 @@ function parseQueryRequest(
);
}

if (queryRequest.query.fields) {
if (
queryRequest.query.fields !== null &&
queryRequest.query.fields !== undefined
) {
requestedFields = getRequestedFieldsFromObject(
collectionObjectBaseType,
collectionObjectType,
Expand All @@ -350,7 +356,10 @@ function parseQueryRequest(
);
}

if (queryRequest.query.aggregates) {
if (
queryRequest.query.aggregates !== null &&
queryRequest.query.aggregates !== undefined
) {
isAggregateQuery = true;
Object.entries(queryRequest.query.aggregates).forEach(
([fieldName, aggregateField]) => {
Expand Down
Loading

0 comments on commit 5478d2a

Please sign in to comment.