From 64440dd0f094066e261244f2bbf2cdb77045ec7c Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Wed, 24 Jul 2024 17:53:30 +0200 Subject: [PATCH] chore(server): Configure native API gateway port via env variable (#8509) --- packages/cubejs-api-gateway/src/gateway.ts | 35 ++++++++++--------- packages/cubejs-api-gateway/src/sql-server.ts | 21 +++++------ .../cubejs-api-gateway/src/types/gateway.ts | 1 + packages/cubejs-backend-shared/src/env.ts | 14 +++++++- .../src/core/optionsValidate.ts | 1 + .../cubejs-server-core/src/core/server.ts | 5 +-- packages/cubejs-server-core/src/core/types.ts | 1 + packages/cubejs-server/src/server.ts | 3 +- 8 files changed, 50 insertions(+), 31 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 59c7ccf060b6f..c3402b3f091f2 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -80,7 +80,7 @@ import { } from './query'; import { cachedHandler } from './cached-handler'; import { createJWKsFetcher } from './jwk'; -import { SQLServer } from './sql-server'; +import { SQLServer, SQLServerConstructorOptions } from './sql-server'; import { getJsonQueryFromGraphQLQuery, makeSchema } from './graphql'; import { ConfigItem, prepareAnnotation } from './helpers/prepareAnnotation'; import transformData from './helpers/transformData'; @@ -160,6 +160,8 @@ class ApiGateway { // eslint-disable-next-line @typescript-eslint/no-unused-vars protected readonly event: (name: string, props?: object) => void; + protected readonly sqlServer: SQLServer; + public constructor( protected readonly apiSecret: string, protected readonly compilerApi: (ctx: RequestContext) => Promise, @@ -192,6 +194,19 @@ class ApiGateway { this.wsContextAcceptor = options.wsContextAcceptor || (() => ({ accepted: true })); // eslint-disable-next-line @typescript-eslint/no-empty-function this.event = options.event || function dummyEvent() {}; + this.sqlServer = this.createSQLServerInstance({ + gatewayPort: options.gatewayPort, + }); + } + + public getSQLServer(): SQLServer { + return this.sqlServer; + } + + protected createSQLServerInstance(options: SQLServerConstructorOptions): SQLServer { + return new SQLServer(this, { + gatewayPort: options.gatewayPort, + }); } public initApp(app: ExpressApplication) { @@ -365,13 +380,11 @@ class ApiGateway { `${this.basePath}/v1/cubesql`, userMiddlewares, userAsyncHandler(async (req, res) => { - const server = this.initSQLServer(); - res.setHeader('Content-Type', 'application/json'); res.setHeader('Transfer-Encoding', 'chunked'); try { - await server.execSql(req.body.query, res, req.context?.securityContext); + await this.sqlServer.execSql(req.body.query, res, req.context?.securityContext); } catch (e: any) { this.handleError({ e, @@ -503,10 +516,8 @@ class ApiGateway { } if (getEnv('nativeApiGateway')) { - const server = this.initSQLServer(); - const proxyMiddleware = createProxyMiddleware({ - target: `http://127.0.0.1:${server.getNativeGatewayPort()}/v2`, + target: `http://127.0.0.1:${this.sqlServer.getNativeGatewayPort()}/v2`, changeOrigin: true, }); @@ -519,16 +530,6 @@ class ApiGateway { app.use(this.handleErrorMiddleware); } - protected _sqlServer: SQLServer | undefined; - - public initSQLServer() { - if (!this._sqlServer) { - this._sqlServer = new SQLServer(this); - } - - return this._sqlServer; - } - public initSubscriptionServer(sendMessage: WebSocketSendMessageFn) { return new SubscriptionServer(this, sendMessage, this.subscriptionStore, this.wsContextAcceptor); } diff --git a/packages/cubejs-api-gateway/src/sql-server.ts b/packages/cubejs-api-gateway/src/sql-server.ts index fd18b4cb52391..81d1d4aff9e9a 100644 --- a/packages/cubejs-api-gateway/src/sql-server.ts +++ b/packages/cubejs-api-gateway/src/sql-server.ts @@ -18,10 +18,14 @@ export type SQLServerOptions = { canSwitchSqlUser?: CanSwitchSQLUserFn, sqlPort?: number, pgSqlPort?: number, - gatewayPort?: number, sqlUser?: string, sqlSuperUser?: string, sqlPassword?: string, + gatewayPort?: number, +}; + +export type SQLServerConstructorOptions = { + gatewayPort?: number, }; export class SQLServer { @@ -31,19 +35,20 @@ export class SQLServer { public constructor( protected readonly apiGateway: ApiGateway, + options: SQLServerConstructorOptions, ) { setupLogger( ({ event }) => apiGateway.log(event), process.env.CUBEJS_LOG_LEVEL === 'trace' ? 'trace' : 'warn' ); - } - public getNativeGatewayPort(): number { - if (this.gatewayPort) { - return this.gatewayPort; + if (options.gatewayPort) { + this.gatewayPort = options.gatewayPort; } + } - throw new Error('Native ApiGateway is not enabled'); + public getNativeGatewayPort(): number { + return this.gatewayPort; } public async execSql(sqlQuery: string, stream: any, securityContext?: any) { @@ -55,10 +60,6 @@ export class SQLServer { throw new Error('Unable to start SQL interface two times'); } - if (options.gatewayPort) { - this.gatewayPort = options.gatewayPort; - } - const checkSqlAuth: CheckSQLAuthFn = (options.checkSqlAuth && this.wrapCheckSqlAuthFn(options.checkSqlAuth)) || this.createDefaultCheckSqlAuthFn(options); diff --git a/packages/cubejs-api-gateway/src/types/gateway.ts b/packages/cubejs-api-gateway/src/types/gateway.ts index 2c0b0bb0043a3..f140105f5fa1e 100644 --- a/packages/cubejs-api-gateway/src/types/gateway.ts +++ b/packages/cubejs-api-gateway/src/types/gateway.ts @@ -49,6 +49,7 @@ type ScheduledRefreshContextsFn = */ interface ApiGatewayOptions { standalone: boolean; + gatewayPort?: number, dataSourceStorage: any; refreshScheduler: any; scheduledRefreshContexts?: ScheduledRefreshContextsFn; diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 61bb508c9ba14..05a5463bcd2a7 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1696,6 +1696,18 @@ const variables: Record any> = { return undefined; }, + nativeApiGatewayPort: () => { + if (process.env.CUBEJS_NATIVE_API_GATEWAY_PORT === 'false') { + return undefined; + } + + const port = asFalseOrPort(process.env.CUBEJS_NATIVE_API_GATEWAY_PORT || 'false', 'CUBEJS_NATIVE_API_GATEWAY_PORT'); + if (port) { + return port; + } + + return undefined; + }, pgSqlPort: () => { if (process.env.CUBEJS_PG_SQL_PORT === 'false') { return undefined; @@ -1727,7 +1739,7 @@ const variables: Record any> = { sqlUser: () => get('CUBEJS_SQL_USER').asString(), sqlPassword: () => get('CUBEJS_SQL_PASSWORD').asString(), sqlSuperUser: () => get('CUBEJS_SQL_SUPER_USER').asString(), - // Internal testing + // Internal testing, please don't enable it. It's not ready for public preview nativeApiGateway: () => get('CUBE_JS_NATIVE_API_GATEWAY_INTERNAL') .asBool(), // Experiments & Preview flags diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts index 8a1dbfe31e5de..7f370fb2fb51e 100644 --- a/packages/cubejs-server-core/src/core/optionsValidate.ts +++ b/packages/cubejs-server-core/src/core/optionsValidate.ts @@ -152,6 +152,7 @@ const schemaOptions = Joi.object().keys({ // SQL API sqlPort: Joi.number(), pgSqlPort: Joi.number(), + gatewayPort: Joi.number(), sqlSuperUser: Joi.string(), checkSqlAuth: Joi.func(), canSwitchSqlUser: Joi.func(), diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index e6415b8e94367..08397e5e833d5 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -426,7 +426,7 @@ export class CubejsServerCore { public initSQLServer() { const apiGateway = this.apiGateway(); - return apiGateway.initSQLServer(); + return apiGateway.getSQLServer(); } protected apiGateway(): ApiGateway { @@ -457,6 +457,7 @@ export class CubejsServerCore { scheduledRefreshTimeZones: this.options.scheduledRefreshTimeZones, serverCoreVersion: this.coreServerVersion, contextToApiScopes: this.options.contextToApiScopes, + gatewayPort: this.options.gatewayPort, event: this.event, } )); @@ -530,7 +531,7 @@ export class CubejsServerCore { this.repository = new FileRepository(this.options.schemaPath); this.repositoryFactory = this.options.repositoryFactory || (() => this.repository); - + this.startScheduledRefreshTimer(); } diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 68fd911a3858b..09d25c5fc6f4b 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -187,6 +187,7 @@ export interface CreateOptions { checkSqlAuth?: CheckSQLAuthFn; canSwitchSqlUser?: CanSwitchSQLUserFn; jwt?: JWTOptions; + gatewayPort?: number; // @deprecated Please use queryRewrite queryTransformer?: QueryRewriteFn; queryRewrite?: QueryRewriteFn; diff --git a/packages/cubejs-server/src/server.ts b/packages/cubejs-server/src/server.ts index c46eeffc4ec0d..eb03b65753d81 100644 --- a/packages/cubejs-server/src/server.ts +++ b/packages/cubejs-server/src/server.ts @@ -63,6 +63,7 @@ export class CubejsServer { webSockets: config.webSockets || getEnv('webSockets'), sqlPort: config.sqlPort || getEnv('sqlPort'), pgSqlPort: config.pgSqlPort || getEnv('pgSqlPort'), + gatewayPort: config.gatewayPort || getEnv('nativeApiGatewayPort'), http: { ...config.http, cors: { @@ -72,7 +73,7 @@ export class CubejsServer { }, }; - this.core = this.createCoreInstance(config, systemOptions); + this.core = this.createCoreInstance(this.config, systemOptions); this.server = null; }