test(S3): improved

src/eckit/io/s3/S3Config.cc

@@ -76,6 +76,8 @@ auto S3Config::make(std::string path) -> std::vector<S3Config> {
         return {};
     }
 
+    LOG_DEBUG_LIB(LibEcKit) << "Reading S3 configuration from: " << configFile << std::endl;
+
     const auto servers = YAMLConfiguration(configFile).getSubConfigurations("servers");
 
     std::vector<S3Config> result;

src/eckit/io/s3/S3Credential.cc

@@ -58,6 +58,8 @@ auto S3Credential::fromFile(std::string path) -> std::vector<S3Credential> {
         return {};
     }
 
+    LOG_DEBUG_LIB(LibEcKit) << "Reading S3 credentials from: " << credFile << std::endl;
+
     std::vector<S3Credential> result;
 
     const auto creds = YAMLConfiguration(credFile).getSubConfigurations("credentials");

tests/io/s3/CMakeLists.txt

@@ -1,23 +1,29 @@
 
 if( eckit_HAVE_S3 )
 
-    file(
-        COPY S3Config.yaml S3Credentials.yaml
-        DESTINATION ${CMAKE_CURRENT_BINARY_DIR}
-    )
-
-    ecbuild_add_test(
-        TARGET      eckit_test_s3client
-        SOURCES     test_s3client.cc
-        INCLUDES    ${AWSSDK_INCLUDE_DIRS}
-        LIBS        eckit ${AWSSDK_LINK_LIBRARIES}
-    )
-
-    ecbuild_add_test(
-        TARGET      eckit_test_s3handle
-        SOURCES     test_s3handle.cc
-        INCLUDES    ${AWSSDK_INCLUDE_DIRS}
-        LIBS        eckit ${AWSSDK_LINK_LIBRARIES}
-    )
+    set( S3_TEST_HOST       "minio" )
+    set( S3_TEST_PORT       "9000" )
+    set( S3_TEST_ENDPOINT   ${S3_TEST_HOST}:${S3_TEST_PORT} )
+    set( S3_TEST_REGION     "eu-central-1" )
+    set( S3_TEST_BUCKET     "eckit-test-bucket" )
+
+    configure_file( test_s3_config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_config.h @ONLY )
+
+    configure_file( S3Config.yaml.in        S3Config.yaml       @ONLY )
+    configure_file( S3Credentials.yaml.in   S3Credentials.yaml  @ONLY )
+
+    set( test_srcs client handle )
+
+    foreach( _test ${test_srcs} )
+
+        ecbuild_add_test(
+            TARGET      eckit_test_s3_${_test}
+            SOURCES     test_s3_${_test}.cc
+            INCLUDES    test_s3_config.h ${AWSSDK_INCLUDE_DIRS}
+            LIBS        eckit ${AWSSDK_LINK_LIBRARIES}
+            ENVIRONMENT "${_test_environment}"
+        )
+
+    endforeach()
 
 endif( eckit_HAVE_S3 )

tests/io/s3/S3Config.yaml → tests/io/s3/S3Config.yaml.in

@@ -1,15 +1,17 @@
 ---
 servers:
+  - endpoint: @S3_TEST_ENDPOINT@
+    region: @S3_TEST_REGION@
+
   - endpoint: "127.0.0.1:9000"
     region: "default" # (default)
-    secure: true   # (default)
-    backend: "AWS" # (default)
-
-  - endpoint: "minio:9000"
-    region: "eu-central-1"
+    secure: true      # (default)
+    backend: "AWS"    # (default)
 
   - endpoint: "localhost:9000"
     region: "eu-central-1"
+    secure: true   # (default)
+    backend: "AWS" # (default)
 
   # region is inferred from the endpoint
   - endpoint: "eu-central-1.ecmwf.int:9000"

tests/io/s3/S3Credentials.yaml.in

@@ -0,0 +1,13 @@
+---
+credentials:
+  - endpoint: @S3_TEST_ENDPOINT@
+    accessKeyID: minio
+    secretKey: minio1234
+
+  - endpoint: localhost:9000
+    accessKeyID: asd1
+    secretKey: asd1
+
+  - endpoint: 127.0.0.1:9000
+    accessKeyID: asd2
+    secretKey: asd2

tests/io/s3/test_s3client.cc → tests/io/s3/test_s3_client.cc

@@ -21,6 +21,7 @@
 #include "eckit/io/s3/S3Session.h"
 #include "eckit/net/Endpoint.h"
 #include "eckit/testing/Test.h"
+#include "test_s3_config.h"
 
 #include <algorithm>
 #include <string>
@@ -37,9 +38,11 @@ namespace eckit::test {
 
 namespace {
 
-const net::Endpoint TEST_ENDPOINT {"minio", 9000};
+const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT};
 
-const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"};
+const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION};
+
+const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"};
 
 bool findString(const std::vector<std::string>& list, const std::string& item) {
     return (std::find(list.begin(), list.end(), item) != list.end());
@@ -61,15 +64,12 @@ void cleanup() {
 
 CASE("s3 client: API") {
 
-    const S3Config     config {TEST_ENDPOINT, "eu-central-1"};
-    const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"};
-
-    EXPECT(S3Session::instance().addClient(config));
-    EXPECT(S3Session::instance().addCredential(cred));
+    EXPECT(S3Session::instance().addClient(TEST_CONFIG));
+    EXPECT(S3Session::instance().addCredential(TEST_CRED));
 
     EXPECT_NO_THROW(cleanup());
 
-    EXPECT_NO_THROW(S3Session::instance().removeClient(config.endpoint));
+    EXPECT_NO_THROW(S3Session::instance().removeClient(TEST_ENDPOINT));
 }
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -95,13 +95,11 @@ CASE("s3 client: read from file") {
 
 CASE("s3 credentials: API") {
 
-    const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"};
-
-    EXPECT(S3Session::instance().addCredential(cred));
+    EXPECT(S3Session::instance().addCredential(TEST_CRED));
 
     EXPECT_NO_THROW(cleanup());
 
-    EXPECT_NO_THROW(S3Session::instance().removeCredential(cred.endpoint));
+    EXPECT_NO_THROW(S3Session::instance().removeCredential(TEST_ENDPOINT));
 }
 
 //----------------------------------------------------------------------------------------------------------------------

tests/io/s3/test_s3_config.h

@@ -0,0 +1,17 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#pragma once
+
+#define S3_TEST_HOST "minio"
+#define S3_TEST_PORT "9000"
+#define S3_TEST_ENDPOINT "minio:9000"
+#define S3_TEST_REGION "eu-central-1"
+#define S3_TEST_BUCKET "eckit-test-bucket"

tests/io/s3/test_s3_config.h.in

@@ -0,0 +1,17 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#pragma once
+
+#cmakedefine S3_TEST_HOST "@S3_TEST_HOST@"
+#cmakedefine S3_TEST_PORT "@S3_TEST_PORT@"
+#cmakedefine S3_TEST_ENDPOINT "@S3_TEST_ENDPOINT@"
+#cmakedefine S3_TEST_REGION "@S3_TEST_REGION@"
+#cmakedefine S3_TEST_BUCKET "@S3_TEST_BUCKET@"

tests/io/s3/test_s3handle.cc → tests/io/s3/test_s3_handle.cc

@@ -26,6 +26,7 @@
 #include "eckit/log/Timer.h"
 #include "eckit/net/Endpoint.h"
 #include "eckit/testing/Test.h"
+#include "test_s3_config.h"
 
 #include <bits/basic_string.h>
 
@@ -49,12 +50,14 @@ namespace {
 
 constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz";
 
-const net::Endpoint TEST_ENDPOINT {"minio", 9000};
-const std::string   TEST_BUCKET {"eckit-s3handle-test-bucket"};
-const std::string   TEST_OBJECT {"eckit-s3handle-test-object"};
+const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"};
+const std::string TEST_OBJECT {"eckit-s3handle-test-object"};
 
-const S3Config     TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"};
-const S3Credential TEST_CREDENTIAL {TEST_ENDPOINT, "minio", "minio1234"};
+const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT};
+
+const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION};
+
+const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"};
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -91,7 +94,7 @@ void cleanup() {
 
 CASE("initialize s3 session") {
 
-    EXPECT(S3Session::instance().addCredential(TEST_CREDENTIAL));
+    EXPECT(S3Session::instance().addCredential(TEST_CRED));
     EXPECT(S3Session::instance().addClient(TEST_CONFIG));
 
     EXPECT_NO_THROW(cleanup());