Merge pull request #3579 from terascope/storage-connection-validation…

…-bug-fix Asset storage connection validation bug fix
terascope · Mar 29, 2024 · b9733c4 · b9733c4
2 parents 8c43999 + 36296f5
commit b9733c4
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 8 deletions.
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "1.1.1",
+    "version": "1.1.2",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "0.58.1",
+    "version": "0.58.2",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {

diff --git a/packages/terafoundation/src/schema.ts b/packages/terafoundation/src/schema.ts
@@ -75,7 +75,7 @@ export function foundationSchema(sysconfig: SysConfig<any>): convict.Schema<any>
             doc: 'Name of the connection type used to store assets',
             default: DEFAULT_ASSET_STORAGE_CONNECTION_TYPE,
             format(connectionTypeName: any): void {
-                const validConnectionTypes = ['elasticsearch-next', 'elasticsearch', 's3'];
+                const validConnectionTypes = ['elasticsearch-next', 's3'];
                 const connectionTypesPresent = Object.keys(sysconfig.terafoundation.connectors);
                 if (!connectionTypesPresent.includes(connectionTypeName)) {
                     throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
@@ -89,15 +89,20 @@ export function foundationSchema(sysconfig: SysConfig<any>): convict.Schema<any>
             doc: 'Name of the connection used to store assets.',
             default: 'default',
             format(connectionName: any): void {
-                let connectionType;
+                let connectionType: string;
                 if (sysconfig.terafoundation.asset_storage_connection_type) {
                     connectionType = sysconfig.terafoundation.asset_storage_connection_type;
                 } else {
                     connectionType = DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
                 }
 
                 const connectionsPresent = Object.keys(sysconfig.terafoundation.connectors[`${connectionType}`]);
-                if (!connectionsPresent.includes(connectionName)) {
+                /// Check to make sure the asset_storage_connection exists inside the connector
+                /// Exclude elasticsearch as this connection type does not utilize this value
+                if (
+                    !connectionsPresent.includes(connectionName)
+                    && !connectionType.includes('elasticsearch-next')
+                ) {
                     throw new Error(`${connectionName} not found in terafoundation.connectors.${connectionType}`);
                 }
             }

diff --git a/packages/terafoundation/test/validate-configs-spec.ts b/packages/terafoundation/test/validate-configs-spec.ts
@@ -386,7 +386,53 @@ describe('Validate Configs', () => {
 
         it('should throw an error', () => {
             expect(() => validateConfigs(cluster as any, config as any, configFile as any))
-                .toThrow('Error validating configuration, caused by Error: asset_storage_connection_type: Invalid asset_storage_connection_type. Valid types: elasticsearch-next,elasticsearch,s3: value was "kafka"');
+                .toThrow('Error validating configuration, caused by Error: asset_storage_connection_type: Invalid asset_storage_connection_type. Valid types: elasticsearch-next,s3: value was "kafka"');
+        });
+    });
+
+    describe('when given a config with an elasticsearch with no default connection', () => {
+        const configFile = {
+            terafoundation: {
+                connectors: {
+                    'elasticsearch-next': {
+                        'not-default': {}
+                    }
+                }
+            },
+            other: {}
+        };
+        const cluster = {
+            isMaster: true,
+        };
+        const config = {
+            config_schema() {
+                return {};
+            }
+        };
+
+        const validatedConfig = validateConfigs(cluster as any, config as any, configFile as any);
+        it('should return valid config', () => {
+            expect(validatedConfig).toMatchObject({
+                terafoundation: {
+                    environment: 'test',
+                    logging: ['console'],
+                    log_level: 'info',
+                    asset_storage_bucket: undefined,
+                    connectors: {
+                        'elasticsearch-next': {
+                            'not-default': {
+                                node: ['http://127.0.0.1:9200'],
+                                sniffOnStart: false,
+                                sniffOnConnectionFault: false,
+                                requestTimeout: 120000,
+                                maxRetries: 3
+                            }
+                        }
+                    },
+                },
+                other: {},
+                _nodeName: os.hostname()
+            });
         });
     });
 });
diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "1.1.1",
+    "version": "1.1.2",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -64,7 +64,7 @@
         "semver": "^7.6.0",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.58.1",
+        "terafoundation": "^0.58.2",
         "uuid": "^9.0.1"
     },
     "devDependencies": {