Bump version to 3.0

.github/workflows/draft-release-notes-workflow.yml

@@ -18,6 +18,6 @@ jobs:
         with:
           config-name: draft-release-notes-config.yml
           tag: (None)
-          version: 2.3.0.0
+          version: 3.0.0.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sql-workbench-test-and-build-workflow.yml

@@ -12,7 +12,7 @@ on:
 env: 
   PLUGIN_NAME: query-workbench-dashboards
   OPENSEARCH_VERSION: 'main'
-  OPENSEARCH_PLUGIN_VERSION: 2.3.0.0
+  OPENSEARCH_PLUGIN_VERSION: 3.0.0.0
 
 jobs:
   build:

build.gradle

@@ -6,9 +6,9 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
         spring_version = "5.3.22"
-        jackson_version = "2.13.3"
+        jackson_version = "2.13.4"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -44,6 +44,8 @@ plugins {
 // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{}
 repositories {
     mavenLocal()
+    // todo. remove this when lucene 9.4.0 is released
+    maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" }
     maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
     mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
 }
@@ -66,6 +68,8 @@ allprojects {
 subprojects {
     repositories {
         mavenLocal()
+        // todo. remove this when lucene 9.4.0 is released
+        maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" }
         maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
         mavenCentral()
     }

doctest/build.gradle

@@ -45,7 +45,9 @@ task stopOpenSearch(type: KillProcessTask)
 doctest.dependsOn startOpenSearch
 doctest.finalizedBy stopOpenSearch
 
-build.dependsOn doctest
+// disable doctest for 3.0 in case no opensearch-ml-3.0 snapshot.
+// https://github.com/opensearch-project/sql/issues/942
+// build.dependsOn doctest
 clean.dependsOn(cleanBootstrap)
 
 // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT

integ-test/build.gradle

@@ -58,6 +58,7 @@ configurations.all {
     resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}"
     resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0"
     resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0"
+    resolutionStrategy.force 'org.slf4j:slf4j-api:1.7.36'
 }
 
 dependencies {
@@ -189,17 +190,18 @@ task compileJdbc(type: Exec) {
     }
 }
 
-String bwcVersion = "1.1.0.0";
+String baseVersion = "2.4.0"
+String bwcVersion = baseVersion + ".0";
 String baseName = "sqlBwcCluster"
 String bwcFilePath = "src/test/resources/bwc/"
 String bwcSqlPlugin = "opensearch-sql-" + bwcVersion + ".zip"
-String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/" + bwcSqlPlugin
+String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcSqlPlugin
 
 2.times { i ->
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["1.1.0", opensearch_version]
+            versions = [baseVersion, opensearch_version]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>() {
                 @Override

integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java

@@ -11,14 +11,15 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 import com.google.common.collect.Maps;
 
+import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.Map;
 import java.util.TimeZone;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
@@ -43,7 +44,7 @@ public class CorrectnessIT extends OpenSearchIntegTestCase {
   private static final Logger LOG = LogManager.getLogger();
 
   @Test
-  public void performComparisonTest() {
+  public void performComparisonTest() throws URISyntaxException {
     TestConfig config = new TestConfig(getCmdLineArgs());
     LOG.info("Starting comparison test {}", config);
 
@@ -73,7 +74,7 @@ private Map<String, String> getCmdLineArgs() {
     return Maps.fromProperties(System.getProperties());
   }
 
-  private DBConnection getThisDBConnection(TestConfig config) {
+  private DBConnection getThisDBConnection(TestConfig config) throws URISyntaxException {
     String dbUrl = config.getDbConnectionUrl();
     if (dbUrl.isEmpty()) {
       return getOpenSearchConnection(config);
@@ -84,7 +85,7 @@ private DBConnection getThisDBConnection(TestConfig config) {
   /**
    * Use OpenSearch cluster given on CLI arg or internal embedded in SQLIntegTestCase
    */
-  private DBConnection getOpenSearchConnection(TestConfig config) {
+  private DBConnection getOpenSearchConnection(TestConfig config) throws URISyntaxException {
     RestClient client;
     String openSearchHost = config.getOpenSearchHostUrl();
     if (openSearchHost.isEmpty()) {

integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java

@@ -17,8 +17,8 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Arrays;
-import org.apache.http.ProtocolVersion;
-import org.apache.http.message.BasicStatusLine;
+import org.apache.hc.core5.http.ProtocolVersion;
+import org.apache.hc.core5.http.message.StatusLine;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -48,7 +48,7 @@ public void setUp() throws IOException {
     Response response = mock(Response.class);
     when(client.performRequest(any(Request.class))).thenReturn(response);
     when(response.getStatusLine())
-        .thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 2, 0), 200, ""));
+        .thenReturn(new StatusLine(new ProtocolVersion("HTTP", 2, 0), 200, ""));
   }
 
   @Test

integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java

@@ -166,13 +166,13 @@ public void percentileTest() throws IOException {
             0.6);
     Assert
         .assertEquals(35.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "75.0"),
-            0.001);
+            0.6);
     Assert
         .assertEquals(39.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "95.0"),
-            0.001);
+            0.6);
     Assert
         .assertEquals(40.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "99.0"),
-            0.001);
+            0.6);
   }
 
   @Test
@@ -183,9 +183,9 @@ public void percentileTestSpecific() throws IOException {
 
     Assert.assertThat(getTotalHits(result), equalTo(1000));
     Assert.assertEquals(25.0,
-        getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.001);
+        getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.6);
     Assert.assertEquals(35.0,
-        getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.001);
+        getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.6);
   }
 
   @Test

integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java

@@ -486,7 +486,36 @@ public void percentileAggregationTest() throws Exception {
 
     List<String> lines = csvResult.getLines();
     Assert.assertEquals(1, lines.size());
-    Assert.assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0));
+
+    List<Double> result =
+        Arrays.stream(lines.get(0).split(","))
+            .mapToDouble(Double::valueOf)
+            .boxed()
+            .collect(Collectors.toList());
+    assertEquals(7, result.size());
+    assertEquals(32.0, result.get(0), 0.6);
+    assertEquals(32.0, result.get(1), 0.6);
+    assertEquals(34.0, result.get(2), 0.6);
+
+    assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0), 0.6);
+  }
+
+  private void assertEquals(String expected, String actual, Double delta) {
+    List<Double> actualList =
+        Arrays.stream(actual.split(","))
+            .mapToDouble(Double::valueOf)
+            .boxed()
+            .collect(Collectors.toList());
+    List<Double> expectedList =
+        Arrays.stream(expected.split(","))
+            .mapToDouble(Double::valueOf)
+            .boxed()
+            .collect(Collectors.toList());
+
+    assertEquals(expectedList.size(), actualList.size());
+    for (int i = 0; i < expectedList.size(); i++) {
+      assertEquals(expectedList.get(i), actualList.get(i), delta);
+    }
   }
 
   @Test

integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java

@@ -35,10 +35,10 @@ public void testPercentilesQuery() {
 
     JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0");
 
-    assertThat(percentileRow.getDouble("25.0"), equalTo(31.5));
-    assertThat(percentileRow.getDouble("50.0"), equalTo(33.5));
-    assertThat(percentileRow.getDouble("75.0"), equalTo(36.5));
-    assertThat(percentileRow.getDouble("99.9"), equalTo(39.0));
+    assertEquals(31.5, percentileRow.getDouble("25.0"), 0.6);
+    assertEquals(33.5, percentileRow.getDouble("50.0"), 0.6);
+    assertEquals(36.5, percentileRow.getDouble("75.0"), 0.6);
+    assertEquals(39.0, percentileRow.getDouble("99.9"), 0.6);
   }
 
   // https://github.com/opensearch-project/sql/issues/537
@@ -53,10 +53,10 @@ public void testSlowQuery() throws IOException {
 
     assertThat(response.getJSONArray("datarows").length(), equalTo(1));
     JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0");
-    assertThat(percentileRow.getDouble("25.0"), equalTo(31.5));
-    assertThat(percentileRow.getDouble("50.0"), equalTo(33.5));
-    assertThat(percentileRow.getDouble("75.0"), equalTo(36.5));
-    assertThat(percentileRow.getDouble("99.9"), equalTo(39.0));
+    assertEquals(31.5, percentileRow.getDouble("25.0"), 0.6);
+    assertEquals(33.5, percentileRow.getDouble("50.0"), 0.6);
+    assertEquals(36.5, percentileRow.getDouble("75.0"), 0.6);
+    assertEquals(39.0, percentileRow.getDouble("99.9"), 0.6);
 
     wipeAllClusterSettings();
   }

integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java

@@ -9,16 +9,20 @@
 import java.io.IOException;
 import java.util.Map;
 import java.util.Optional;
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.ssl.SSLContextBuilder;
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.opensearch.client.Request;
@@ -56,7 +60,7 @@ protected String getProtocol() {
   protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
     RestClientBuilder builder = RestClient.builder(hosts);
     if (isHttps()) {
-      configureHttpsClient(builder, settings);
+      configureHttpsClient(builder, settings, hosts[0]);
     } else {
       configureClient(builder, settings);
     }
@@ -68,19 +72,24 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
   protected static void wipeAllOpenSearchIndices() throws IOException {
     // include all the indices, included hidden indices.
     // https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html#cat-indices-api-query-params
-    Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
-    JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8"));
-    for (Object object : jsonArray) {
-      JSONObject jsonObject = (JSONObject) object;
-      String indexName = jsonObject.getString("index");
-      //.opendistro_security isn't allowed to delete from cluster
-      if (!indexName.startsWith(".opensearch_dashboards") && !indexName.startsWith(".opendistro")) {
-        client().performRequest(new Request("DELETE", "/" + indexName));
+    try {
+      Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
+      JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8"));
+      for (Object object : jsonArray) {
+        JSONObject jsonObject = (JSONObject) object;
+        String indexName = jsonObject.getString("index");
+        //.opendistro_security isn't allowed to delete from cluster
+        if (!indexName.startsWith(".opensearch_dashboards") && !indexName.startsWith(".opendistro")) {
+          client().performRequest(new Request("DELETE", "/" + indexName));
+        }
       }
+    } catch (ParseException e) {
+      throw new IOException(e);
     }
   }
 
-  protected static void configureHttpsClient(RestClientBuilder builder, Settings settings)
+  protected static void configureHttpsClient(RestClientBuilder builder, Settings settings,
+                                             HttpHost httpHost)
       throws IOException {
     Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
     Header[] defaultHeaders = new Header[headers.size()];
@@ -94,15 +103,21 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s
           .orElseThrow(() -> new RuntimeException("user name is missing"));
       String password = Optional.ofNullable(System.getProperty("password"))
           .orElseThrow(() -> new RuntimeException("password is missing"));
-      CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+      BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
       credentialsProvider
-          .setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
+          .setCredentials(new AuthScope(httpHost), new UsernamePasswordCredentials(userName,
+              password.toCharArray()));
       try {
-        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
-            //disable the certificate since our testing cluster just uses the default security configuration
-            .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
-            .setSSLContext(SSLContextBuilder.create()
+        final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
+            .setSslContext(SSLContextBuilder.create()
                 .loadTrustMaterial(null, (chains, authType) -> true)
+                .build())
+            .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+            .build();
+
+        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
+            .setConnectionManager(PoolingAsyncClientConnectionManagerBuilder.create()
+                .setTlsStrategy(tlsStrategy)
                 .build());
       } catch (Exception e) {
         throw new RuntimeException(e);
@@ -114,7 +129,7 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s
         TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString,
             CLIENT_SOCKET_TIMEOUT);
     builder.setRequestConfigCallback(
-        conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
+        conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()))));
     if (settings.hasValue(CLIENT_PATH_PREFIX)) {
       builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
     }

legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java

@@ -106,7 +106,8 @@ public class LocalClusterStateTest {
         "      }\n" +
         "    },\n" +
         "    \"mapping_version\": \"1\",\n" +
-        "    \"settings_version\": \"1\"\n" +
+        "    \"settings_version\": \"1\",\n" +
+        "    \"aliases_version\": \"1\"\n" +
         //=======================================================
         "  }\n" +
         "}";

legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java

@@ -189,7 +189,8 @@ public static void stubMockClient(Client mockClient) {
                 "      }\n" +
                 "    },\n" +
                 "    \"mapping_version\": \"1\",\n" +
-                "    \"settings_version\": \"1\"\n" +
+                "    \"settings_version\": \"1\",\n" +
+                "    \"aliases_version\": \"1\"\n" +
                 //=======================================================
                 "  }\n" +
                 "}";

legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java

@@ -83,7 +83,8 @@ public class MultipleIndexClusterUtils {
             "      }\n" +
             "    },\n" +
             "    \"mapping_version\": \"1\",\n" +
-            "    \"settings_version\": \"1\"\n" +
+            "    \"settings_version\": \"1\",\n" +
+            "    \"aliases_version\": \"1\"\n" +
             "  }\n" +
             "}";
 
@@ -134,7 +135,8 @@ public class MultipleIndexClusterUtils {
             "      }\n" +
             "    },\n" +
             "    \"mapping_version\": \"1\",\n" +
-            "    \"settings_version\": \"1\"\n" +
+            "    \"settings_version\": \"1\",\n" +
+            "    \"aliases_version\": \"1\"\n" +
             "  }\n" +
             "}";