From f771ab1c9edfdeee2988e26424f8794226bac061 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 25 Dec 2024 11:49:02 +0100 Subject: [PATCH] test(S3): improved common config --- tests/io/s3/CMakeLists.txt | 3 +- tests/io/s3/test_s3_client.cc | 64 +++++++------------- tests/io/s3/test_s3_config.h | 64 +++++++++++++++++++- tests/io/s3/test_s3_config.h.in | 62 ++++++++++++++++++++ tests/io/s3/test_s3_handle.cc | 100 +++++++++----------------------- 5 files changed, 176 insertions(+), 117 deletions(-) diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt index 8ade180f1..069af9f09 100644 --- a/tests/io/s3/CMakeLists.txt +++ b/tests/io/s3/CMakeLists.txt @@ -5,7 +5,8 @@ if( eckit_HAVE_S3 ) set( S3_TEST_PORT "9000" ) set( S3_TEST_ENDPOINT ${S3_TEST_HOST}:${S3_TEST_PORT} ) set( S3_TEST_REGION "eu-central-1" ) - set( S3_TEST_BUCKET "eckit-test-bucket" ) + set( S3_TEST_BUCKET "eckit-test-s3-bucket" ) + set( S3_TEST_OBJECT "eckit-test-s3-object" ) configure_file( test_s3_config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_config.h @ONLY ) diff --git a/tests/io/s3/test_s3_client.cc b/tests/io/s3/test_s3_client.cc index b5ff7f3c3..142e1ac89 100644 --- a/tests/io/s3/test_s3_client.cc +++ b/tests/io/s3/test_s3_client.cc @@ -16,14 +16,12 @@ #include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Config.h" -#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" #include "test_s3_config.h" -#include #include #include @@ -36,40 +34,18 @@ namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- -namespace { - -const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; - -const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; - -const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; - -bool findString(const std::vector& list, const std::string& item) { - return (std::find(list.begin(), list.end(), item) != list.end()); -} - -void cleanup() { - auto client = S3Client::makeUnique(TEST_CONFIG); - for (const auto& name : {S3BucketPath {"test-bucket-1"}, S3BucketPath {{"test-bucket-2"}}}) { - if (client->bucketExists(name)) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } -} - -} // namespace +const std::vector testBuckets {"test-bucket-1", "test-bucket-2"}; //---------------------------------------------------------------------------------------------------------------------- CASE("s3 client: API") { - EXPECT(S3Session::instance().addClient(TEST_CONFIG)); - EXPECT(S3Session::instance().addCredential(TEST_CRED)); + EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - EXPECT_NO_THROW(S3Session::instance().removeClient(TEST_ENDPOINT)); + EXPECT_NO_THROW(S3Session::instance().removeClient(s3::TEST_ENDPOINT)); } //---------------------------------------------------------------------------------------------------------------------- @@ -95,11 +71,11 @@ CASE("s3 client: read from file") { CASE("s3 credentials: API") { - EXPECT(S3Session::instance().addCredential(TEST_CRED)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - EXPECT_NO_THROW(S3Session::instance().removeCredential(TEST_ENDPOINT)); + EXPECT_NO_THROW(S3Session::instance().removeCredential(s3::TEST_ENDPOINT)); } //---------------------------------------------------------------------------------------------------------------------- @@ -124,7 +100,7 @@ CASE("s3 credentials: read from file") { //---------------------------------------------------------------------------------------------------------------------- CASE("s3 backends") { - S3Config cfgTmp(TEST_CONFIG); + S3Config cfgTmp(s3::TEST_CONFIG); cfgTmp.backend = S3Backend::AWS; EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); @@ -148,12 +124,12 @@ CASE("s3 backends") { //---------------------------------------------------------------------------------------------------------------------- CASE("create s3 bucket in non-existing region") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); // this test requires an S3 endpoint that sets it's region // a MinIO instance with empty region will not throw an exception - auto cfgTmp = TEST_CONFIG; + auto cfgTmp = s3::TEST_CONFIG; cfgTmp.region = "non-existing-region-random"; EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket({"test-bucket-1"})); @@ -162,9 +138,9 @@ CASE("create s3 bucket in non-existing region") { //---------------------------------------------------------------------------------------------------------------------- CASE("create s3 bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - auto client = S3Client::makeUnique(TEST_CONFIG); + auto client = S3Client::makeUnique(s3::TEST_CONFIG); EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); @@ -176,17 +152,17 @@ CASE("create s3 bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("list s3 buckets") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - auto client = S3Client::makeUnique(TEST_CONFIG); + auto client = S3Client::makeUnique(s3::TEST_CONFIG); EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); EXPECT_NO_THROW(client->createBucket({"test-bucket-2"})); { const auto buckets = client->listBuckets(); - EXPECT(findString(buckets, "test-bucket-1")); - EXPECT(findString(buckets, {"test-bucket-2"})); + EXPECT(s3::findString(buckets, "test-bucket-1")); + EXPECT(s3::findString(buckets, {"test-bucket-2"})); EXPECT_NO_THROW(client->deleteBucket({"test-bucket-1"})); EXPECT_NO_THROW(client->deleteBucket({"test-bucket-2"})); @@ -194,8 +170,8 @@ CASE("list s3 buckets") { { const auto buckets = client->listBuckets(); - EXPECT_NOT(findString(buckets, "test-bucket-1")); - EXPECT_NOT(findString(buckets, {"test-bucket-2"})); + EXPECT_NOT(s3::findString(buckets, "test-bucket-1")); + EXPECT_NOT(s3::findString(buckets, {"test-bucket-2"})); } EXPECT_THROWS(client->deleteBucket({"test-bucket-1"})); @@ -211,7 +187,7 @@ int main(int argc, char** argv) { ret = run_tests(argc, argv); - test::cleanup(); + test::s3::cleanup(test::testBuckets); return ret; } diff --git a/tests/io/s3/test_s3_config.h b/tests/io/s3/test_s3_config.h index e5333149d..004d1320e 100644 --- a/tests/io/s3/test_s3_config.h +++ b/tests/io/s3/test_s3_config.h @@ -10,8 +10,70 @@ #pragma once +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + #define S3_TEST_HOST "minio" #define S3_TEST_PORT "9000" #define S3_TEST_ENDPOINT "minio:9000" #define S3_TEST_REGION "eu-central-1" -#define S3_TEST_BUCKET "eckit-test-bucket" +#define S3_TEST_BUCKET "eckit-test-s3-bucket" +#define S3_TEST_OBJECT "eckit-test-s3-object" + +namespace eckit::test::s3 { + +//---------------------------------------------------------------------------------------------------------------------- + +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; + +inline bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +inline void cleanup(const std::vector& bucketPaths) { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto& path : bucketPaths) { + if (client->bucketExists(path)) { + client->emptyBucket(path); + client->deleteBucket(path); + } + } +} + +void writePerformance(S3BucketName& bucket, const int count) { + eckit::Timer timer; + + Buffer buffer(1024 * 1024); + buffer.zero(); + + timer.start(); + for (int i = 0; i < count; i++) { + const auto objName = S3_TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(buffer.data(), buffer.size()); + } + timer.stop(); + + std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count + << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test::s3 diff --git a/tests/io/s3/test_s3_config.h.in b/tests/io/s3/test_s3_config.h.in index 4b67ff8e5..3d05cdc84 100644 --- a/tests/io/s3/test_s3_config.h.in +++ b/tests/io/s3/test_s3_config.h.in @@ -10,8 +10,70 @@ #pragma once +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + #cmakedefine S3_TEST_HOST "@S3_TEST_HOST@" #cmakedefine S3_TEST_PORT "@S3_TEST_PORT@" #cmakedefine S3_TEST_ENDPOINT "@S3_TEST_ENDPOINT@" #cmakedefine S3_TEST_REGION "@S3_TEST_REGION@" #cmakedefine S3_TEST_BUCKET "@S3_TEST_BUCKET@" +#cmakedefine S3_TEST_OBJECT "@S3_TEST_OBJECT@" + +namespace eckit::test::s3 { + +//---------------------------------------------------------------------------------------------------------------------- + +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; + +inline bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +inline void cleanup(const std::vector& bucketPaths) { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto& path : bucketPaths) { + if (client->bucketExists(path)) { + client->emptyBucket(path); + client->deleteBucket(path); + } + } +} + +void writePerformance(S3BucketName& bucket, const int count) { + eckit::Timer timer; + + Buffer buffer(1024 * 1024); + buffer.zero(); + + timer.start(); + for (int i = 0; i < count; i++) { + const auto objName = S3_TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(buffer.data(), buffer.size()); + } + timer.stop(); + + std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count + << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test::s3 diff --git a/tests/io/s3/test_s3_handle.cc b/tests/io/s3/test_s3_handle.cc index 868634e66..bf61df6df 100644 --- a/tests/io/s3/test_s3_handle.cc +++ b/tests/io/s3/test_s3_handle.cc @@ -15,15 +15,11 @@ #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/URI.h" -#include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3BucketName.h" #include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3Session.h" -#include "eckit/log/Bytes.h" -#include "eckit/log/Timer.h" #include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" #include "test_s3_config.h" @@ -31,11 +27,10 @@ #include #include -#include #include -#include #include #include +#include using namespace std::string_literals; @@ -46,58 +41,21 @@ namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- -namespace { - -constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz"; +constexpr std::string_view TEST_DATA {"abcdefghijklmnopqrstuvwxyz"}; const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; -const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; - -const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; - -const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; - -//---------------------------------------------------------------------------------------------------------------------- - -void writePerformance(S3BucketName& bucket, const int count) { - eckit::Timer timer; - - Buffer buffer(1024 * 1024); - buffer.zero(); - - timer.start(); - for (int i = 0; i < count; i++) { - const auto objName = TEST_OBJECT + std::to_string(i); - bucket.makeObject(objName)->put(buffer.data(), buffer.size()); - } - timer.stop(); - - std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count - << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; -} - -void cleanup() { - auto client = S3Client::makeUnique(TEST_CONFIG); - for (const auto& name : {TEST_BUCKET}) { - if (client->bucketExists(name)) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } -} - -} // namespace +const std::vector testBuckets {TEST_BUCKET}; //---------------------------------------------------------------------------------------------------------------------- CASE("initialize s3 session") { - EXPECT(S3Session::instance().addCredential(TEST_CRED)); - EXPECT(S3Session::instance().addClient(TEST_CONFIG)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); + EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG)); - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); } CASE("invalid s3 bucket") { @@ -109,9 +67,9 @@ CASE("invalid s3 bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: no bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -129,9 +87,9 @@ CASE("S3BucketName: no bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: create bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -147,9 +105,9 @@ CASE("S3BucketName: create bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: empty bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); @@ -165,9 +123,9 @@ CASE("S3BucketName: empty bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: bucket with object") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); @@ -186,14 +144,14 @@ CASE("S3BucketName: bucket with object") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: basic operations") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3Client::makeUnique(TEST_CONFIG)->createBucket(TEST_BUCKET); + S3Client::makeUnique(s3::TEST_CONFIG)->createBucket(TEST_BUCKET); const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { S3ObjectName object(uri); @@ -234,9 +192,9 @@ CASE("S3Handle: basic operations") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: openForRead") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -259,9 +217,9 @@ CASE("S3Handle: openForRead") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: openForWrite") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { // NO BUCKET std::unique_ptr handle(uri.newWriteHandle()); @@ -282,9 +240,9 @@ CASE("S3Handle: openForWrite") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: read") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -316,19 +274,19 @@ CASE("S3Handle: read") { //---------------------------------------------------------------------------------------------------------------------- CASE("s3 performance: write 1 10 100 objects") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET); S3BucketName bucket(uri); EXPECT_NO_THROW(bucket.ensureCreated()); - writePerformance(bucket, 1); + s3::writePerformance(bucket, 1); - writePerformance(bucket, 10); + s3::writePerformance(bucket, 10); - writePerformance(bucket, 100); + s3::writePerformance(bucket, 100); } //---------------------------------------------------------------------------------------------------------------------- @@ -340,7 +298,7 @@ int main(int argc, char** argv) { ret = run_tests(argc, argv); - test::cleanup(); + test::s3::cleanup(test::testBuckets); return ret; }