diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml index 4e970066..5a919631 100644 --- a/analytics-datastore-clickhouse/docker-compose.cluster.yml +++ b/analytics-datastore-clickhouse/docker-compose.cluster.yml @@ -5,6 +5,8 @@ services: image: ${CLICKHOUSE_IMAGE} ulimits: noFile: 262144 + environment: + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} volumes: - clickhouse-data-01:/var/lib/clickhouse/ hostname: analytics-datastore-clickhouse-01 @@ -33,6 +35,7 @@ services: source: clickhouse_trace_log.xml networks: public: + reverse-proxy: default: analytics-datastore-clickhouse-02: @@ -44,6 +47,8 @@ services: - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}" ulimits: noFile: 262144 + environment: + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} volumes: - clickhouse-data-02:/var/lib/clickhouse/ configs: @@ -78,6 +83,8 @@ services: - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}" ulimits: noFile: 262144 + environment: + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} volumes: - clickhouse-data-03:/var/lib/clickhouse/ configs: @@ -108,6 +115,8 @@ services: hostname: analytics-datastore-clickhouse-04 ulimits: noFile: 262144 + environment: + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} volumes: - clickhouse-data-04:/var/lib/clickhouse/ configs: @@ -213,4 +222,7 @@ networks: public: name: clickhouse_public external: true + reverse-proxy: + name: reverse-proxy_public + external: true default: diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml index 3f175532..0c00e47c 100644 --- a/analytics-datastore-clickhouse/docker-compose.yml +++ b/analytics-datastore-clickhouse/docker-compose.yml @@ -5,6 +5,8 @@ services: image: ${CLICKHOUSE_IMAGE} ulimits: noFile: 262144 + environment: + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} volumes: - clickhouse-data:/var/lib/clickhouse/ configs: @@ -18,6 +20,7 @@ services: source: clickhouse_trace_log.xml networks: public: + reverse-proxy: default: volumes: @@ -49,4 +52,7 @@ networks: public: name: clickhouse_public external: true + reverse-proxy: + name: reverse-proxy_public + external: true default: diff --git a/analytics-datastore-clickhouse/importer/config/clickhouseConfig.js b/analytics-datastore-clickhouse/importer/config/clickhouseConfig.js index 224e129d..847f1476 100755 --- a/analytics-datastore-clickhouse/importer/config/clickhouseConfig.js +++ b/analytics-datastore-clickhouse/importer/config/clickhouseConfig.js @@ -7,11 +7,16 @@ const CLICKHOUSE_HOST = process.env.CLICKHOUSE_HOST || 'analytics-datastore-clickhouse'; const CLICKHOUSE_PORT = parseInt(process.env.CLICKHOUSE_PORT || '8123'); const CLICKHOUSE_DEBUG = Boolean(process.env.CLICKHOUSE_DEBUG || false); +const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || ''; const clickhouse = new ClickHouse({ url: CLICKHOUSE_HOST, port: CLICKHOUSE_PORT, debug: CLICKHOUSE_DEBUG, + basicAuth: { + username: 'default', + password: CLICKHOUSE_PASSWORD, + }, raw: true, }); diff --git a/analytics-datastore-clickhouse/importer/docker-compose.config.yml b/analytics-datastore-clickhouse/importer/docker-compose.config.yml index a2dc6781..cac1b2a9 100644 --- a/analytics-datastore-clickhouse/importer/docker-compose.config.yml +++ b/analytics-datastore-clickhouse/importer/docker-compose.config.yml @@ -9,6 +9,7 @@ services: environment: CLICKHOUSE_HOST: ${CLICKHOUSE_HOST} CLICKHOUSE_PORT: ${CLICKHOUSE_PORT} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} CLUSTERED_MODE: ${CLUSTERED_MODE} configs: - source: config-importer-clickhouseConfig.js diff --git a/analytics-datastore-clickhouse/package-metadata.json b/analytics-datastore-clickhouse/package-metadata.json index a7b0efb7..cd8d853f 100644 --- a/analytics-datastore-clickhouse/package-metadata.json +++ b/analytics-datastore-clickhouse/package-metadata.json @@ -8,6 +8,7 @@ "environmentVariables": { "CLICKHOUSE_HOST": "analytics-datastore-clickhouse", "CLICKHOUSE_PORT": "8123", + "CLICKHOUSE_PASSWORD": "dev_password_only", "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6", "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1", "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2", diff --git a/dashboard-visualiser-superset/importer/config/superset-export.zip b/dashboard-visualiser-superset/importer/config/superset-export.zip index b0cad1bb..da83f169 100644 Binary files a/dashboard-visualiser-superset/importer/config/superset-export.zip and b/dashboard-visualiser-superset/importer/config/superset-export.zip differ diff --git a/dashboard-visualiser-superset/importer/config/supersetConfig.js b/dashboard-visualiser-superset/importer/config/supersetConfig.js index c516453d..0e6207b1 100644 --- a/dashboard-visualiser-superset/importer/config/supersetConfig.js +++ b/dashboard-visualiser-superset/importer/config/supersetConfig.js @@ -14,6 +14,8 @@ const SUPERSET_LOGIN_PATH = process.env.SUPERSET_LOGIN_PATH || '/api/v1/security/login'; const SUPERSET_IMPORT_PATH = process.env.SUPERSET_IMPORT_PATH || '/api/v1/assets/import/'; +const SUPERSET_DATABASE_PUT_PATH = + process.env.SUPERSET_DATABASE_PUT_PATH || '/api/v1/database'; const CONFIG_FILE = process.env.CONFIG_FILE; const SUPERSET_SSL = process.env.SUPERSET_SSL || 'false'; @@ -27,7 +29,7 @@ const getAccessToken = async () => { refresh: true, }); - var config = { + const config = { method: 'POST', url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_LOGIN_PATH}`, headers: { @@ -47,39 +49,81 @@ const getAccessToken = async () => { } }; +const importZipConfig = async (accessToken) => { + const data = new FormData(); + + if (CONFIG_FILE) { + data.append( + 'bundle', + fs.createReadStream(path.resolve(__dirname, CONFIG_FILE)) + ); + + const config = { + method: 'POST', + url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_IMPORT_PATH}`, + headers: { + 'Content-Type': 'application/zip', + Authorization: `Bearer ${accessToken}`, + ...data.getHeaders(), + }, + data: data, + }; + + const res = await axios(config); + + console.log('\n', res.data); + console.log('\nConfig imported successfully. exit.'); + } else { + throw new Error( + '\nNo path was provided. Please provide the path of the config.' + ); + } +} + +const replaceClickhouseConnectionString = async (accessToken) => { + const CLICKHOUSE_HOST = process.env.CLICKHOUSE_HOST || 'analytics-datastore-clickhouse'; + const CLICKHOUSE_PORT = process.env.CLICKHOUSE_PORT || '8123'; + const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || 'dev_password_only'; + + const databaseConfig = { + allow_ctas: false, + allow_cvas: false, + allow_dml: false, + allow_file_upload: false, + allow_run_async: false, + cache_timeout: 0, + configuration_method: "sqlalchemy_form", + database_name: "Clickhouse connection", + driver: "connect", + engine: "clickhousedb", + expose_in_sqllab: true, + sqlalchemy_uri: `clickhousedb+connect://default:${CLICKHOUSE_PASSWORD}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/default`, + uuid: "868ecd6d-f099-46ab-a100-dd91173bc63f" + }; + + const config = { + method: 'POST', + url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_DATABASE_PUT_PATH}`, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${accessToken}` + }, + data: databaseConfig, + }; + + const res = await axios(config); + + console.log('\n', res.data); + console.log('\Database connection updated successfully. exit.'); +} + (async () => { try { const accessToken = await getAccessToken(); if (accessToken) { - var data = new FormData(); - - if (CONFIG_FILE) { - data.append( - 'bundle', - fs.createReadStream(path.resolve(__dirname, CONFIG_FILE)) - ); - - var config = { - method: 'POST', - url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_IMPORT_PATH}`, - headers: { - 'Content-Type': 'application/zip', - Authorization: `Bearer ${accessToken}`, - ...data.getHeaders(), - }, - data: data, - }; - - const res = await axios(config); - - console.log('\n', res.data); - console.log('\nConfig imported successfully. exit.'); - } else { - throw new Error( - '\nNo path was provided. Please provide the path of the config.' - ); - } + await importZipConfig(accessToken); + await replaceClickhouseConnectionString(accessToken); } else { throw new Error('\nNo access token was generated.'); } diff --git a/dashboard-visualiser-superset/importer/docker-compose.config.yml b/dashboard-visualiser-superset/importer/docker-compose.config.yml index 352a36e6..455aa005 100644 --- a/dashboard-visualiser-superset/importer/docker-compose.config.yml +++ b/dashboard-visualiser-superset/importer/docker-compose.config.yml @@ -13,12 +13,17 @@ services: SUPERSET_API_USERNAME: ${SUPERSET_USERNAME} CONFIG_FILE: ${SUPERSET_CONFIG_FILE} SUPERSET_SSL: ${SUPERSET_SSL} + CLICKHOUSE_HOST: ${CLICKHOUSE_HOST} + CLICKHOUSE_PORT: ${CLICKHOUSE_PORT} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} configs: - source: config-importer-supersetConfig.js target: /supersetConfig.js - source: config-importer-superset-export.zip target: /superset-export.zip command: sh -c "cd / && npm i axios form-data && node /supersetConfig.js" + networks: + clickhouse: configs: config-importer-supersetConfig.js: @@ -31,3 +36,8 @@ configs: name: config-importer-superset-export.zip-${config_importer_superset_export_zip_DIGEST:?err} labels: name: superset + +networks: + clickhouse: + name: clickhouse_public + external: true diff --git a/dashboard-visualiser-superset/package-metadata.json b/dashboard-visualiser-superset/package-metadata.json index a62e15ea..570d64cd 100644 --- a/dashboard-visualiser-superset/package-metadata.json +++ b/dashboard-visualiser-superset/package-metadata.json @@ -33,6 +33,9 @@ "SUPERSET_POSTGRESQL_PASSWORD": "instant101", "SUPERSET_POSTGRESQL_DATABASE": "superset", "SUPERSET_POSTGRESQL_URL": "postgres-1:5432", + "CLICKHOUSE_HOST": "analytics-datastore-clickhouse", + "CLICKHOUSE_PORT": 8123, + "CLICKHOUSE_PASSWORD": "dev_password_only", "SUPERSET_TRAEFIK_SUBDOMAIN": "superset" } } diff --git a/test/cucumber/features/single-mode/superset.feature b/test/cucumber/features/single-mode/superset.feature index 4666504f..a0059820 100644 --- a/test/cucumber/features/single-mode/superset.feature +++ b/test/cucumber/features/single-mode/superset.feature @@ -2,19 +2,19 @@ Feature: Dashboard Visualiser Superset? Does the Dashboard Visualiser Superset package work as expected Scenario: Init Dashboard Visualiser Superset - Given I use parameters "package init -n=dashboard-visualiser-superset --only --dev --env-file=.env.local" + Given I use parameters "package init -n=dashboard-visualiser-superset --dev --env-file=.env.local" When I launch the platform with params Then The service "dashboard-visualiser-superset" should be started with 1 replica And The service "postgres-metastore" should be started with 1 replica And The service "superset-config-importer" should be removed - And The service "ddashboard-visualiser-superset" should have healthy containers + And The service "dashboard-visualiser-superset" should have healthy containers And The service "dashboard-visualiser-superset" should be connected to the networks | reverse-proxy_public | clickhouse_public | keycloak_public | superset_default | And There should be 2 service - And There should be 2 volumes + And There should be 3 volumes Scenario: Destroy Dashboard Visualiser Superset - Given I use parameters "package destroy -n=dashboard-visualiser-superset --only --dev --env-file=.env.local" + Given I use parameters "package destroy -n=dashboard-visualiser-superset --dev --env-file=.env.local" When I launch the platform with params Then The service "dashboard-visualiser-superset" should be removed And There should be 0 service