From 25b69611f815af957f9244459279b4079d53a4c4 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 14 Sep 2022 11:51:32 -0700 Subject: [PATCH 01/12] Bump version to 3.0 Signed-off-by: Peng Huo --- .github/workflows/draft-release-notes-workflow.yml | 2 +- .github/workflows/sql-odbc-release-workflow.yml | 2 +- .github/workflows/sql-workbench-release-workflow.yml | 2 +- .github/workflows/sql-workbench-test-and-build-workflow.yml | 2 +- build.gradle | 2 +- workbench/opensearch_dashboards.json | 4 ++-- workbench/package.json | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 660a8a1a51..506430817f 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 2.3.0.0 + version: 3.0.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 0d08865378..47edb778bd 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -12,7 +12,7 @@ env: ODBC_BUILD_PATH: "./build/odbc/build" AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" PLUGIN_NAME: opensearch-sql-odbc - OD_VERSION: 2.3.0.0 + OD_VERSION: 3.0.0.0 jobs: build-mac: diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index 840428e538..ddf91915c8 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -8,7 +8,7 @@ on: env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.3.0.0 + OPENSEARCH_PLUGIN_VERSION: 3.0.0.0 jobs: diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index d4da17bf7f..34bab63f58 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.3.0.0 + OPENSEARCH_PLUGIN_VERSION: 3.0.0.0 jobs: diff --git a/build.gradle b/build.gradle index c96655a5c1..fa6796ef50 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") spring_version = "5.3.22" jackson_version = "2.13.3" isSnapshot = "true" == System.getProperty("build.snapshot", "true") diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index 79aefec25f..972bedd4ef 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", - "version": "2.3.0.0", - "opensearchDashboardsVersion": "2.3.0", + "version": "3.0.0.0", + "opensearchDashboardsVersion": "3.0.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index 2fddbb9937..a7a3c727d1 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,6 +1,6 @@ { "name": "opensearch-query-workbench", - "version": "2.3.0.0", + "version": "3.0.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", From 34a420fb6adc7147a68aa9539f69b8fdda1ac947 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 14 Sep 2022 13:23:30 -0700 Subject: [PATCH 02/12] add lucene repo Signed-off-by: Peng Huo --- build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index fa6796ef50..0d9b6fc00a 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,8 @@ plugins { // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} repositories { mavenLocal() + // todo. remove this when lucene 9.4.0 is released + maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available } @@ -66,6 +68,8 @@ allprojects { subprojects { repositories { mavenLocal() + // todo. remove this when lucene 9.4.0 is released + maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() } From 58370533201feb1b14d871572dd8beb04fb67b1e Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 29 Sep 2022 19:45:35 -0700 Subject: [PATCH 03/12] Fix failed IT Signed-off-by: Peng Huo --- .../opensearch/sql/legacy/AggregationIT.java | 10 +++--- .../sql/legacy/CsvFormatResponseIT.java | 31 ++++++++++++++++++- .../org/opensearch/sql/legacy/JdbcTestIT.java | 16 +++++----- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index 1d530efa5e..5c9cd6943c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -166,13 +166,13 @@ public void percentileTest() throws IOException { 0.6); Assert .assertEquals(35.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "75.0"), - 0.001); + 0.6); Assert .assertEquals(39.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "95.0"), - 0.001); + 0.6); Assert .assertEquals(40.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "99.0"), - 0.001); + 0.6); } @Test @@ -183,9 +183,9 @@ public void percentileTestSpecific() throws IOException { Assert.assertThat(getTotalHits(result), equalTo(1000)); Assert.assertEquals(25.0, - getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.001); + getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.6); Assert.assertEquals(35.0, - getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.001); + getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.6); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index 9a08302577..d562794409 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -486,7 +486,36 @@ public void percentileAggregationTest() throws Exception { List lines = csvResult.getLines(); Assert.assertEquals(1, lines.size()); - Assert.assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0)); + + List result = + Arrays.stream(lines.get(0).split(",")) + .mapToDouble(Double::valueOf) + .boxed() + .collect(Collectors.toList()); + assertEquals(7, result.size()); + assertEquals(32.0, result.get(0), 0.6); + assertEquals(32.0, result.get(1), 0.6); + assertEquals(34.0, result.get(2), 0.6); + + assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0), 0.6); + } + + private void assertEquals(String expected, String actual, Double delta) { + List actualList = + Arrays.stream(actual.split(",")) + .mapToDouble(Double::valueOf) + .boxed() + .collect(Collectors.toList()); + List expectedList = + Arrays.stream(expected.split(",")) + .mapToDouble(Double::valueOf) + .boxed() + .collect(Collectors.toList()); + + assertEquals(expectedList.size(), actualList.size()); + for (int i = 0; i < expectedList.size(); i++) { + assertEquals(expectedList.get(i), actualList.get(i), delta); + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java index 74f999f368..c50a03f596 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java @@ -35,10 +35,10 @@ public void testPercentilesQuery() { JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0"); - assertThat(percentileRow.getDouble("25.0"), equalTo(31.5)); - assertThat(percentileRow.getDouble("50.0"), equalTo(33.5)); - assertThat(percentileRow.getDouble("75.0"), equalTo(36.5)); - assertThat(percentileRow.getDouble("99.9"), equalTo(39.0)); + assertEquals(31.5, percentileRow.getDouble("25.0"), 0.6); + assertEquals(33.5, percentileRow.getDouble("50.0"), 0.6); + assertEquals(36.5, percentileRow.getDouble("75.0"), 0.6); + assertEquals(39.0, percentileRow.getDouble("99.9"), 0.6); } // https://github.com/opensearch-project/sql/issues/537 @@ -53,10 +53,10 @@ public void testSlowQuery() throws IOException { assertThat(response.getJSONArray("datarows").length(), equalTo(1)); JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0"); - assertThat(percentileRow.getDouble("25.0"), equalTo(31.5)); - assertThat(percentileRow.getDouble("50.0"), equalTo(33.5)); - assertThat(percentileRow.getDouble("75.0"), equalTo(36.5)); - assertThat(percentileRow.getDouble("99.9"), equalTo(39.0)); + assertEquals(31.5, percentileRow.getDouble("25.0"), 0.6); + assertEquals(33.5, percentileRow.getDouble("50.0"), 0.6); + assertEquals(36.5, percentileRow.getDouble("75.0"), 0.6); + assertEquals(39.0, percentileRow.getDouble("99.9"), 0.6); wipeAllClusterSettings(); } From 42ad02fa11c8b9e2043a568c1ed90c75667aa638 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 30 Sep 2022 10:14:04 -0700 Subject: [PATCH 04/12] bump jackson to 2.13.4 Signed-off-by: Peng Huo --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0d9b6fc00a..42be5c8b84 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") spring_version = "5.3.22" - jackson_version = "2.13.3" + jackson_version = "2.13.4" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') From e94113146e86f14bacd42c0524f0115ed5a014ee Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 30 Sep 2022 12:03:24 -0700 Subject: [PATCH 05/12] bump bwc version to 2.3.0 Signed-off-by: Peng Huo --- integ-test/build.gradle | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 5e0a53bf1a..d5ca1bab15 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -189,17 +189,18 @@ task compileJdbc(type: Exec) { } } -String bwcVersion = "1.1.0.0"; +String baseVersion = "2.3.0" +String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" String bwcSqlPlugin = "opensearch-sql-" + bwcVersion + ".zip" -String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/" + bwcSqlPlugin +String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/6039/linux/x64/tar/builds/opensearch/plugins/" + bwcSqlPlugin 2.times { i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["1.1.0", opensearch_version] + versions = [baseVersion, opensearch_version] numberOfNodes = 3 plugin(provider(new Callable() { @Override From 00305d6c1f8be465399e0564a90835edccd006d5 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 3 Oct 2022 13:01:02 -0700 Subject: [PATCH 06/12] bump bwc version to 2.4.0 Signed-off-by: Peng Huo --- integ-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d5ca1bab15..bd6e34a836 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -189,7 +189,7 @@ task compileJdbc(type: Exec) { } } -String baseVersion = "2.3.0" +String baseVersion = "2.4.0" String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" From 429448961cfe5ff89f402c927612f79e6e3c0ed4 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 3 Oct 2022 13:54:10 -0700 Subject: [PATCH 07/12] Update SQL 2.4.0.0 link Signed-off-by: Peng Huo --- integ-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index bd6e34a836..b102602dc9 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -194,7 +194,7 @@ String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" String bwcSqlPlugin = "opensearch-sql-" + bwcVersion + ".zip" -String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/6039/linux/x64/tar/builds/opensearch/plugins/" + bwcSqlPlugin +String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcSqlPlugin 2.times { i -> testClusters { From 02364e056cd6a35dcd5d1eca5975a37dddc672af Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 12 Oct 2022 23:38:30 +0000 Subject: [PATCH 08/12] add groupid Signed-off-by: Peng Huo --- plugin/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugin/build.gradle b/plugin/build.gradle index c1aae613bd..ce362f08d7 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -55,6 +55,7 @@ publishing { pom { name = 'opensearch-sql' description = 'OpenSearch SQL' + groupId = "org.opensearch.plugin" licenses { license { name = 'The Apache License, Version 2.0' @@ -90,6 +91,7 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + resolutionStrategy.force 'org.slf4j:slf4j-api:1.7.36' } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) From 76abbd00c4b28cdee9ade85941c9d2062b04704e Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 12 Oct 2022 19:39:31 -0700 Subject: [PATCH 09/12] update IT for http 5 Signed-off-by: Peng Huo --- integ-test/build.gradle | 1 + .../sql/correctness/CorrectnessIT.java | 9 +-- .../tests/OpenSearchConnectionTest.java | 6 +- .../sql/legacy/OpenSearchSQLRestTestCase.java | 69 +++++++++++-------- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index b102602dc9..b9630c8211 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -58,6 +58,7 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + resolutionStrategy.force 'org.slf4j:slf4j-api:1.7.36' } dependencies { diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java b/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java index a6f3e561b3..889f60f4ad 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.collect.Maps; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -18,7 +19,7 @@ import java.util.Date; import java.util.Map; import java.util.TimeZone; -import org.apache.http.HttpHost; +import org.apache.hc.core5.http.HttpHost; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONObject; @@ -43,7 +44,7 @@ public class CorrectnessIT extends OpenSearchIntegTestCase { private static final Logger LOG = LogManager.getLogger(); @Test - public void performComparisonTest() { + public void performComparisonTest() throws URISyntaxException { TestConfig config = new TestConfig(getCmdLineArgs()); LOG.info("Starting comparison test {}", config); @@ -73,7 +74,7 @@ private Map getCmdLineArgs() { return Maps.fromProperties(System.getProperties()); } - private DBConnection getThisDBConnection(TestConfig config) { + private DBConnection getThisDBConnection(TestConfig config) throws URISyntaxException { String dbUrl = config.getDbConnectionUrl(); if (dbUrl.isEmpty()) { return getOpenSearchConnection(config); @@ -84,7 +85,7 @@ private DBConnection getThisDBConnection(TestConfig config) { /** * Use OpenSearch cluster given on CLI arg or internal embedded in SQLIntegTestCase */ - private DBConnection getOpenSearchConnection(TestConfig config) { + private DBConnection getOpenSearchConnection(TestConfig config) throws URISyntaxException { RestClient client; String openSearchHost = config.getOpenSearchHostUrl(); if (openSearchHost.isEmpty()) { diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java index 73659525f2..5b33884814 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java @@ -17,8 +17,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; -import org.apache.http.ProtocolVersion; -import org.apache.http.message.BasicStatusLine; +import org.apache.hc.core5.http.ProtocolVersion; +import org.apache.hc.core5.http.message.StatusLine; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -48,7 +48,7 @@ public void setUp() throws IOException { Response response = mock(Response.class); when(client.performRequest(any(Request.class))).thenReturn(response); when(response.getStatusLine()) - .thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 2, 0), 200, "")); + .thenReturn(new StatusLine(new ProtocolVersion("HTTP", 2, 0), 200, "")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 0369ee715a..72daa286e8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -9,16 +9,20 @@ import java.io.IOException; import java.util.Map; import java.util.Optional; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.message.BasicHeader; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.util.EntityUtils; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.client.Request; @@ -56,7 +60,7 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { - configureHttpsClient(builder, settings); + configureHttpsClient(builder, settings, hosts[0]); } else { configureClient(builder, settings); } @@ -68,19 +72,24 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE protected static void wipeAllOpenSearchIndices() throws IOException { // include all the indices, included hidden indices. // https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html#cat-indices-api-query-params - Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); - JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); - for (Object object : jsonArray) { - JSONObject jsonObject = (JSONObject) object; - String indexName = jsonObject.getString("index"); - //.opendistro_security isn't allowed to delete from cluster - if (!indexName.startsWith(".opensearch_dashboards") && !indexName.startsWith(".opendistro")) { - client().performRequest(new Request("DELETE", "/" + indexName)); + try { + Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); + for (Object object : jsonArray) { + JSONObject jsonObject = (JSONObject) object; + String indexName = jsonObject.getString("index"); + //.opendistro_security isn't allowed to delete from cluster + if (!indexName.startsWith(".opensearch_dashboards") && !indexName.startsWith(".opendistro")) { + client().performRequest(new Request("DELETE", "/" + indexName)); + } } + } catch (ParseException e) { + throw new IOException(e); } } - protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings, + HttpHost httpHost) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); Header[] defaultHeaders = new Header[headers.size()]; @@ -94,15 +103,21 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s .orElseThrow(() -> new RuntimeException("user name is missing")); String password = Optional.ofNullable(System.getProperty("password")) .orElseThrow(() -> new RuntimeException("password is missing")); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider - .setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + .setCredentials(new AuthScope(httpHost), new UsernamePasswordCredentials(userName, + password.toCharArray())); try { - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - //disable the certificate since our testing cluster just uses the default security configuration - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(SSLContextBuilder.create() + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(SSLContextBuilder.create() .loadTrustMaterial(null, (chains, authType) -> true) + .build()) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager(PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) .build()); } catch (Exception e) { throw new RuntimeException(e); @@ -114,7 +129,7 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback( - conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())))); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); } From 243bd9e9135a96eaf85e844791cf69b6c2f81a9e Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 12 Oct 2022 22:26:36 -0700 Subject: [PATCH 10/12] add alias version Signed-off-by: Peng Huo --- opensearch/src/test/resources/mappings/accounts.json | 5 +++-- opensearch/src/test/resources/mappings/accounts2.json | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/opensearch/src/test/resources/mappings/accounts.json b/opensearch/src/test/resources/mappings/accounts.json index a7024b6490..5431568304 100644 --- a/opensearch/src/test/resources/mappings/accounts.json +++ b/opensearch/src/test/resources/mappings/accounts.json @@ -87,6 +87,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} diff --git a/opensearch/src/test/resources/mappings/accounts2.json b/opensearch/src/test/resources/mappings/accounts2.json index d300b8c523..0a5937fd36 100644 --- a/opensearch/src/test/resources/mappings/accounts2.json +++ b/opensearch/src/test/resources/mappings/accounts2.json @@ -88,6 +88,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} From 6f33801a3211c28217f2e20eabe3f0845cf795af Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 20 Oct 2022 09:11:54 -0700 Subject: [PATCH 11/12] add alias version Signed-off-by: Peng Huo --- .../sql/legacy/unittest/LocalClusterStateTest.java | 3 ++- .../org/opensearch/sql/legacy/util/CheckScriptContents.java | 3 ++- .../sql/legacy/util/MultipleIndexClusterUtils.java | 6 ++++-- legacy/src/test/resources/mappings/field_mappings.json | 5 +++-- legacy/src/test/resources/mappings/semantics.json | 5 +++-- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java index 67ad4b9ba2..b23b24413a 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java @@ -106,7 +106,8 @@ public class LocalClusterStateTest { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + //======================================================= " }\n" + "}"; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java index f427170cfc..0f9ee88c69 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java @@ -189,7 +189,8 @@ public static void stubMockClient(Client mockClient) { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + //======================================================= " }\n" + "}"; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java index 0213cac01b..1a3ee07f88 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java @@ -83,7 +83,8 @@ public class MultipleIndexClusterUtils { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + " }\n" + "}"; @@ -134,7 +135,8 @@ public class MultipleIndexClusterUtils { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + " }\n" + "}"; diff --git a/legacy/src/test/resources/mappings/field_mappings.json b/legacy/src/test/resources/mappings/field_mappings.json index bf059d0ae9..997e794d1b 100644 --- a/legacy/src/test/resources/mappings/field_mappings.json +++ b/legacy/src/test/resources/mappings/field_mappings.json @@ -67,6 +67,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} diff --git a/legacy/src/test/resources/mappings/semantics.json b/legacy/src/test/resources/mappings/semantics.json index 46de8f2eaa..e6e7be958e 100644 --- a/legacy/src/test/resources/mappings/semantics.json +++ b/legacy/src/test/resources/mappings/semantics.json @@ -87,6 +87,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} From 7047062a2c86571a54a2ccc3cc3a90b9c11eebdc Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 20 Oct 2022 09:19:55 -0700 Subject: [PATCH 12/12] disable doctest Signed-off-by: Peng Huo --- doctest/build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doctest/build.gradle b/doctest/build.gradle index eec5a791b8..60b6fd7464 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -45,7 +45,9 @@ task stopOpenSearch(type: KillProcessTask) doctest.dependsOn startOpenSearch doctest.finalizedBy stopOpenSearch -build.dependsOn doctest +// disable doctest for 3.0 in case no opensearch-ml-3.0 snapshot. +// https://github.com/opensearch-project/sql/issues/942 +// build.dependsOn doctest clean.dependsOn(cleanBootstrap) // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT