Merge pull request #3383 from terascope/store-schema-chagnes

Store schema chagnes
terascope · Jul 26, 2023 · e92f6a5 · e92f6a5
2 parents f682a18 + fcd5f53
commit e92f6a5
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 55 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -54,6 +54,8 @@ jobs:
   teraslice-elasticsearch-tests:
     runs-on: ubuntu-latest
     strategy:
+      # opensearch is finiky, keep testing others if it fails
+      fail-fast: false
       matrix:
         node-version: [14.21.3, 16.19.1, 18.16.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
@@ -86,6 +88,8 @@ jobs:
   elasticsearch-store-tests:
     runs-on: ubuntu-latest
     strategy:
+      # opensearch is finiky, keep testing others if it fails
+      fail-fast: false
       matrix:
         node-version: [14.21.3, 16.19.1, 18.16.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
@@ -147,6 +151,8 @@ jobs:
   elasticsearch-api-tests:
     runs-on: ubuntu-latest
     strategy:
+      # opensearch is finiky, keep testing others if it fails
+      fail-fast: false
       matrix:
         node-version: [14.21.3, 16.19.1, 18.16.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
@@ -209,6 +215,8 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     strategy:
+      # opensearch is finiky, keep testing others if it fails
+      fail-fast: false
       matrix:
         node-version: [14.21.3, 16.19.1, 18.16.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.86.1",
+    "version": "0.86.2",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

diff --git a/packages/scripts/package.json b/packages/scripts/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.53.0",
+    "version": "0.53.1",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {

diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts
@@ -34,7 +34,7 @@ export const KAFKA_NAME = process.env.KAFKA_NAME || 'kafka';
 export const KAFKA_HOSTNAME = process.env.KAFKA_HOSTNAME || HOST_IP;
 export const KAFKA_PORT = process.env.KAFKA_PORT || '49092';
 export const KAFKA_BROKER = `${KAFKA_HOSTNAME}:${KAFKA_PORT}`;
-export const KAFKA_VERSION = process.env.KAFKA_VERSION || '2.3';
+export const KAFKA_VERSION = process.env.KAFKA_VERSION || '3.1';
 export const KAFKA_DOCKER_IMAGE = process.env.KAFKA_DOCKER_IMAGE || 'blacktop/kafka';
 
 export const MINIO_NAME = process.env.MINIO_NAME || 'minio';

diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "0.45.3",
+    "version": "0.45.4",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {

diff --git a/packages/terafoundation/src/connectors/s3.ts b/packages/terafoundation/src/connectors/s3.ts
@@ -50,12 +50,12 @@ export default {
                 default: '',
                 format: String
             },
-            s3ForcePathStyle: {
+            forcePathStyle: {
                 doc: '',
                 default: false,
                 format: Boolean
             },
-            s3BucketEndpoint: {
+            bucketEndpoint: {
                 doc: '',
                 default: false,
                 format: Boolean

diff --git a/packages/terafoundation/test/validate-configs-spec.ts b/packages/terafoundation/test/validate-configs-spec.ts
@@ -148,8 +148,8 @@ describe('Validate Configs', () => {
                                 maxRedirects: 10,
                                 maxRetries: 3,
                                 region: 'us-east-1',
-                                s3BucketEndpoint: false,
-                                s3ForcePathStyle: false,
+                                bucketEndpoint: false,
+                                forcePathStyle: false,
                                 secretAccessKey: null,
                                 sslEnabled: true,
                             }

diff --git a/packages/teraslice/lib/storage/backends/mappings/ex.json b/packages/teraslice/lib/storage/backends/mappings/ex.json
@@ -43,51 +43,6 @@
                 "_slicer_stats": {
                     "type": "object"
                 },
-                "_slicer_stats.workers_available": {
-                    "type": "integer"
-                },
-                "_slicer_stats.workers_active": {
-                    "type": "integer"
-                },
-                "_slicer_stats.workers_joined": {
-                    "type": "integer"
-                },
-                "_slicer_stats.workers_reconnected": {
-                    "type": "integer"
-                },
-                "_slicer_stats.workers_disconnected": {
-                    "type": "integer"
-                },
-                "_slicer_stats.job_duration": {
-                    "type": "integer"
-                },
-                "_slicer_stats.failed": {
-                    "type": "integer"
-                },
-                "_slicer_stats.subslices": {
-                    "type": "integer"
-                },
-                "_slicer_stats.queued": {
-                    "type": "integer"
-                },
-                "_slicer_stats.slice_range_expansion": {
-                    "type": "integer"
-                },
-                "_slicer_stats.processed": {
-                    "type": "integer"
-                },
-                "_slicer_stats.slicers": {
-                    "type": "integer"
-                },
-                "_slicer_stats.subslice_by_key": {
-                    "type": "integer"
-                },
-                "_slicer_stats.started": {
-                    "type": "date"
-                },
-                "_slicer_stats.queuing_complete": {
-                    "type": "date"
-                },
                 "_created": {
                     "type": "date"
                 },

diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.86.1",
+    "version": "0.86.2",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -65,7 +65,7 @@
         "semver": "^7.3.8",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.45.3",
+        "terafoundation": "^0.45.4",
         "uuid": "^9.0.0"
     },
     "devDependencies": {