Skip to content

Commit

Permalink
test(S3): improved common config [skip ci]
Browse files Browse the repository at this point in the history
  • Loading branch information
@mcakircali
mcakircali committed Dec 25, 2024
1 parent 2cc56af commit e1fccb9
Show file tree
Hide file tree
Showing 5 changed files with 176 additions and 117 deletions.
3 changes: 2 additions & 1 deletion tests/io/s3/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ if( eckit_HAVE_S3 )
set( S3_TEST_PORT "9000" )
set( S3_TEST_ENDPOINT ${S3_TEST_HOST}:${S3_TEST_PORT} )
set( S3_TEST_REGION "eu-central-1" )
set( S3_TEST_BUCKET "eckit-test-bucket" )
set( S3_TEST_BUCKET "eckit-test-s3-bucket" )
set( S3_TEST_OBJECT "eckit-test-s3-object" )

configure_file( test_s3_config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_config.h @ONLY )

Expand Down
64 changes: 20 additions & 44 deletions tests/io/s3/test_s3_client.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,12 @@
#include "eckit/io/s3/S3BucketPath.h"
#include "eckit/io/s3/S3Client.h"
#include "eckit/io/s3/S3Config.h"
#include "eckit/io/s3/S3Credential.h"
#include "eckit/io/s3/S3Exception.h"
#include "eckit/io/s3/S3Session.h"
#include "eckit/net/Endpoint.h"
#include "eckit/testing/Test.h"
#include "test_s3_config.h"

#include <algorithm>
#include <string>
#include <vector>

Expand All @@ -36,40 +34,18 @@ namespace eckit::test {

//----------------------------------------------------------------------------------------------------------------------

namespace {

const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT};

const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION};

const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"};

bool findString(const std::vector<std::string>& list, const std::string& item) {
return (std::find(list.begin(), list.end(), item) != list.end());
}

void cleanup() {
auto client = S3Client::makeUnique(TEST_CONFIG);
for (const auto& name : {S3BucketPath {"test-bucket-1"}, S3BucketPath {{"test-bucket-2"}}}) {
if (client->bucketExists(name)) {
client->emptyBucket(name);
client->deleteBucket(name);
}
}
}

} // namespace
const std::vector<std::string> testBuckets {"test-bucket-1", "test-bucket-2"};

//----------------------------------------------------------------------------------------------------------------------

CASE("s3 client: API") {

EXPECT(S3Session::instance().addClient(TEST_CONFIG));
EXPECT(S3Session::instance().addCredential(TEST_CRED));
EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG));
EXPECT(S3Session::instance().addCredential(s3::TEST_CRED));

EXPECT_NO_THROW(cleanup());
EXPECT_NO_THROW(s3::cleanup(testBuckets));

EXPECT_NO_THROW(S3Session::instance().removeClient(TEST_ENDPOINT));
EXPECT_NO_THROW(S3Session::instance().removeClient(s3::TEST_ENDPOINT));
}

//----------------------------------------------------------------------------------------------------------------------
Expand All @@ -95,11 +71,11 @@ CASE("s3 client: read from file") {

CASE("s3 credentials: API") {

EXPECT(S3Session::instance().addCredential(TEST_CRED));
EXPECT(S3Session::instance().addCredential(s3::TEST_CRED));

EXPECT_NO_THROW(cleanup());
EXPECT_NO_THROW(s3::cleanup(testBuckets));

EXPECT_NO_THROW(S3Session::instance().removeCredential(TEST_ENDPOINT));
EXPECT_NO_THROW(S3Session::instance().removeCredential(s3::TEST_ENDPOINT));
}

//----------------------------------------------------------------------------------------------------------------------
Expand All @@ -124,7 +100,7 @@ CASE("s3 credentials: read from file") {
//----------------------------------------------------------------------------------------------------------------------

CASE("s3 backends") {
S3Config cfgTmp(TEST_CONFIG);
S3Config cfgTmp(s3::TEST_CONFIG);

cfgTmp.backend = S3Backend::AWS;
EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp));
Expand All @@ -148,12 +124,12 @@ CASE("s3 backends") {
//----------------------------------------------------------------------------------------------------------------------

CASE("create s3 bucket in non-existing region") {
EXPECT_NO_THROW(cleanup());
EXPECT_NO_THROW(s3::cleanup(testBuckets));

// this test requires an S3 endpoint that sets it's region
// a MinIO instance with empty region will not throw an exception

auto cfgTmp = TEST_CONFIG;
auto cfgTmp = s3::TEST_CONFIG;
cfgTmp.region = "non-existing-region-random";

EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket({"test-bucket-1"}));
Expand All @@ -162,9 +138,9 @@ CASE("create s3 bucket in non-existing region") {
//----------------------------------------------------------------------------------------------------------------------

CASE("create s3 bucket") {
EXPECT_NO_THROW(cleanup());
EXPECT_NO_THROW(s3::cleanup(testBuckets));

auto client = S3Client::makeUnique(TEST_CONFIG);
auto client = S3Client::makeUnique(s3::TEST_CONFIG);

EXPECT_NO_THROW(client->createBucket({"test-bucket-1"}));

Expand All @@ -176,26 +152,26 @@ CASE("create s3 bucket") {
//----------------------------------------------------------------------------------------------------------------------

CASE("list s3 buckets") {
EXPECT_NO_THROW(cleanup());
EXPECT_NO_THROW(s3::cleanup(testBuckets));

auto client = S3Client::makeUnique(TEST_CONFIG);
auto client = S3Client::makeUnique(s3::TEST_CONFIG);
EXPECT_NO_THROW(client->createBucket({"test-bucket-1"}));
EXPECT_NO_THROW(client->createBucket({"test-bucket-2"}));

{
const auto buckets = client->listBuckets();

EXPECT(findString(buckets, "test-bucket-1"));
EXPECT(findString(buckets, {"test-bucket-2"}));
EXPECT(s3::findString(buckets, "test-bucket-1"));
EXPECT(s3::findString(buckets, {"test-bucket-2"}));

EXPECT_NO_THROW(client->deleteBucket({"test-bucket-1"}));
EXPECT_NO_THROW(client->deleteBucket({"test-bucket-2"}));
}

{
const auto buckets = client->listBuckets();
EXPECT_NOT(findString(buckets, "test-bucket-1"));
EXPECT_NOT(findString(buckets, {"test-bucket-2"}));
EXPECT_NOT(s3::findString(buckets, "test-bucket-1"));
EXPECT_NOT(s3::findString(buckets, {"test-bucket-2"}));
}

EXPECT_THROWS(client->deleteBucket({"test-bucket-1"}));
Expand All @@ -211,7 +187,7 @@ int main(int argc, char** argv) {

ret = run_tests(argc, argv);

test::cleanup();
test::s3::cleanup(test::testBuckets);

return ret;
}
64 changes: 63 additions & 1 deletion tests/io/s3/test_s3_config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,70 @@

#pragma once

#include "eckit/io/Buffer.h"
#include "eckit/io/s3/S3BucketName.h"
#include "eckit/io/s3/S3BucketPath.h"
#include "eckit/io/s3/S3Client.h"
#include "eckit/io/s3/S3Config.h"
#include "eckit/io/s3/S3Credential.h"
#include "eckit/io/s3/S3ObjectName.h"
#include "eckit/log/Bytes.h"
#include "eckit/log/Timer.h"
#include "eckit/net/Endpoint.h"

#include <algorithm>
#include <iostream>
#include <string>
#include <vector>

#define S3_TEST_HOST "minio"
#define S3_TEST_PORT "9000"
#define S3_TEST_ENDPOINT "minio:9000"
#define S3_TEST_REGION "eu-central-1"
#define S3_TEST_BUCKET "eckit-test-bucket"
#define S3_TEST_BUCKET "eckit-test-s3-bucket"
#define S3_TEST_OBJECT "eckit-test-s3-object"

namespace eckit::test::s3 {

//----------------------------------------------------------------------------------------------------------------------

const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT};

const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION};

const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"};

inline bool findString(const std::vector<std::string>& list, const std::string& item) {
return (std::find(list.begin(), list.end(), item) != list.end());
}

inline void cleanup(const std::vector<std::string>& bucketPaths) {
auto client = S3Client::makeUnique(TEST_CONFIG);
for (const auto& path : bucketPaths) {
if (client->bucketExists(path)) {
client->emptyBucket(path);
client->deleteBucket(path);
}
}
}

void writePerformance(S3BucketName& bucket, const int count) {
eckit::Timer timer;

Buffer buffer(1024 * 1024);
buffer.zero();

timer.start();
for (int i = 0; i < count; i++) {
const auto objName = S3_TEST_OBJECT + std::to_string(i);
bucket.makeObject(objName)->put(buffer.data(), buffer.size());
}
timer.stop();

std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count
<< " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl;
}

//----------------------------------------------------------------------------------------------------------------------

} // namespace eckit::test::s3
62 changes: 62 additions & 0 deletions tests/io/s3/test_s3_config.h.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,70 @@

#pragma once

#include "eckit/io/Buffer.h"
#include "eckit/io/s3/S3BucketName.h"
#include "eckit/io/s3/S3BucketPath.h"
#include "eckit/io/s3/S3Client.h"
#include "eckit/io/s3/S3Config.h"
#include "eckit/io/s3/S3Credential.h"
#include "eckit/io/s3/S3ObjectName.h"
#include "eckit/log/Bytes.h"
#include "eckit/log/Timer.h"
#include "eckit/net/Endpoint.h"

#include <algorithm>
#include <iostream>
#include <string>
#include <vector>

#cmakedefine S3_TEST_HOST "@S3_TEST_HOST@"
#cmakedefine S3_TEST_PORT "@S3_TEST_PORT@"
#cmakedefine S3_TEST_ENDPOINT "@S3_TEST_ENDPOINT@"
#cmakedefine S3_TEST_REGION "@S3_TEST_REGION@"
#cmakedefine S3_TEST_BUCKET "@S3_TEST_BUCKET@"
#cmakedefine S3_TEST_OBJECT "@S3_TEST_OBJECT@"

namespace eckit::test::s3 {

//----------------------------------------------------------------------------------------------------------------------

const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT};

const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION};

const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"};

inline bool findString(const std::vector<std::string>& list, const std::string& item) {
return (std::find(list.begin(), list.end(), item) != list.end());
}

inline void cleanup(const std::vector<std::string>& bucketPaths) {
auto client = S3Client::makeUnique(TEST_CONFIG);
for (const auto& path : bucketPaths) {
if (client->bucketExists(path)) {
client->emptyBucket(path);
client->deleteBucket(path);
}
}
}

void writePerformance(S3BucketName& bucket, const int count) {
eckit::Timer timer;

Buffer buffer(1024 * 1024);
buffer.zero();

timer.start();
for (int i = 0; i < count; i++) {
const auto objName = S3_TEST_OBJECT + std::to_string(i);
bucket.makeObject(objName)->put(buffer.data(), buffer.size());
}
timer.stop();

std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count
<< " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl;
}

//----------------------------------------------------------------------------------------------------------------------

} // namespace eckit::test::s3
Loading

0 comments on commit e1fccb9

Please sign in to comment.