From 3647ce8de3d9e96d1c6cd508de9478ac3b550201 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 17 Dec 2024 21:56:38 +0100 Subject: [PATCH] feat(S3): add config and credentials from file fix several minor issues --- src/eckit/CMakeLists.txt | 57 ++++---- src/eckit/io/s3/S3BucketName.cc | 37 ++--- src/eckit/io/s3/S3BucketName.h | 12 +- src/eckit/io/s3/S3Client.cc | 33 ++--- src/eckit/io/s3/S3Client.h | 42 ++++-- src/eckit/io/s3/S3Config.cc | 103 +++++++++++-- src/eckit/io/s3/S3Config.h | 64 ++++++-- src/eckit/io/s3/S3Credential.cc | 72 +++++++++ src/eckit/io/s3/S3Credential.h | 38 ++++- src/eckit/io/s3/S3Exception.cc | 24 ++- src/eckit/io/s3/S3Exception.h | 21 +-- src/eckit/io/s3/S3Name.cc | 30 +++- src/eckit/io/s3/S3Name.h | 31 ++-- src/eckit/io/s3/S3ObjectName.cc | 27 ++-- src/eckit/io/s3/S3ObjectName.h | 8 +- src/eckit/io/s3/S3Session.cc | 125 ++++++++-------- src/eckit/io/s3/S3Session.h | 44 ++++-- src/eckit/io/s3/S3URIManager.cc | 11 +- src/eckit/io/s3/S3URIManager.h | 16 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 124 +++++++++------- src/eckit/io/s3/aws/S3ClientAWS.h | 31 ++-- src/eckit/io/s3/aws/S3ContextAWS.cc | 2 + tests/io/CMakeLists.txt | 14 +- tests/io/s3/CMakeLists.txt | 23 +++ tests/io/s3/S3Config.yaml | 15 ++ tests/io/s3/S3Credentials.yaml | 13 ++ tests/io/s3/test_s3client.cc | 218 ++++++++++++++++++++++++++++ tests/io/{ => s3}/test_s3handle.cc | 163 +++++++++++++-------- tests/io/test_s3client.cc | 135 ----------------- 29 files changed, 1023 insertions(+), 510 deletions(-) create mode 100644 src/eckit/io/s3/S3Credential.cc create mode 100644 tests/io/s3/CMakeLists.txt create mode 100644 tests/io/s3/S3Config.yaml create mode 100644 tests/io/s3/S3Credentials.yaml create mode 100644 tests/io/s3/test_s3client.cc rename tests/io/{ => s3}/test_s3handle.cc (58%) delete mode 100644 tests/io/test_s3client.cc diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index d2817cc0b..7242598db 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -283,33 +283,34 @@ if(HAVE_RADOS) ) endif() -if(HAVE_AWS_S3) -list( APPEND eckit_io_srcs -io/s3/aws/S3ClientAWS.cc -io/s3/aws/S3ClientAWS.h -io/s3/aws/S3ContextAWS.cc -io/s3/aws/S3ContextAWS.h -io/s3/S3BucketName.cc -io/s3/S3BucketName.h -io/s3/S3Client.cc -io/s3/S3Client.h -io/s3/S3Config.cc -io/s3/S3Config.h -io/s3/S3Credential.h -io/s3/S3Exception.cc -io/s3/S3Exception.h -io/s3/S3Handle.cc -io/s3/S3Handle.h -io/s3/S3Name.cc -io/s3/S3Name.h -io/s3/S3ObjectName.cc -io/s3/S3ObjectName.h -io/s3/S3Session.cc -io/s3/S3Session.h -io/s3/S3URIManager.cc -io/s3/S3URIManager.h -) -endif(HAVE_AWS_S3) +if( eckit_HAVE_AWS_S3 ) + list( APPEND eckit_io_srcs + io/s3/aws/S3ClientAWS.cc + io/s3/aws/S3ClientAWS.h + io/s3/aws/S3ContextAWS.cc + io/s3/aws/S3ContextAWS.h + io/s3/S3BucketName.cc + io/s3/S3BucketName.h + io/s3/S3Client.cc + io/s3/S3Client.h + io/s3/S3Config.cc + io/s3/S3Config.h + io/s3/S3Credential.cc + io/s3/S3Credential.h + io/s3/S3Exception.cc + io/s3/S3Exception.h + io/s3/S3Handle.cc + io/s3/S3Handle.h + io/s3/S3Name.cc + io/s3/S3Name.h + io/s3/S3ObjectName.cc + io/s3/S3ObjectName.h + io/s3/S3Session.cc + io/s3/S3Session.h + io/s3/S3URIManager.cc + io/s3/S3URIManager.h + ) +endif( eckit_HAVE_AWS_S3 ) list( APPEND eckit_filesystem_srcs filesystem/BasePathName.cc @@ -960,9 +961,9 @@ ecbuild_add_library( "${BZIP2_INCLUDE_DIRS}" "${AEC_INCLUDE_DIRS}" "${RADOS_INCLUDE_DIRS}" - "${AWSSDK_INCLUDE_DIRS}" "${OPENSSL_INCLUDE_DIR}" "${AIO_INCLUDE_DIRS}" + "${AWSSDK_INCLUDE_DIRS}" PRIVATE_LIBS "${SNAPPY_LIBRARIES}" diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index b921b095b..f90adfabc 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -19,18 +19,27 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Name.h" #include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3BucketName::S3BucketName(const URI& uri): S3Name(uri) { +S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) { parse(); } -S3BucketName::S3BucketName(const net::Endpoint& endpoint, const std::string& bucket): - S3Name(endpoint, "/"), bucket_(bucket) { } +S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket) + : S3Name(endpoint, "/"), bucket_ {std::move(bucket)} { } //---------------------------------------------------------------------------------------------------------------------- @@ -56,38 +65,32 @@ auto S3BucketName::makeObject(const std::string& object) const -> std::unique_pt } auto S3BucketName::exists() const -> bool { - return client()->bucketExists(bucket_); + return client().bucketExists(bucket_); } void S3BucketName::create() { - client()->createBucket(bucket_); + client().createBucket(bucket_); } void S3BucketName::destroy() { - client()->deleteBucket(bucket_); + client().deleteBucket(bucket_); } void S3BucketName::ensureCreated() { try { create(); - } - catch (S3EntityAlreadyExists& e) { - LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; - } + } catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } void S3BucketName::ensureDestroyed() { try { - client()->emptyBucket(bucket_); - client()->deleteBucket(bucket_); - } - catch (S3EntityNotFound& e) { - LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; - } + client().emptyBucket(bucket_); + client().deleteBucket(bucket_); + } catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } auto S3BucketName::listObjects() const -> std::vector { - return client()->listObjects(bucket_); + return client().listObjects(bucket_); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index 57fbb50cf..fafbab992 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -21,6 +21,12 @@ #pragma once #include "eckit/io/s3/S3Name.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include namespace eckit { @@ -28,11 +34,11 @@ class S3ObjectName; //---------------------------------------------------------------------------------------------------------------------- -class S3BucketName: public S3Name { +class S3BucketName : public S3Name { public: // methods explicit S3BucketName(const URI& uri); - S3BucketName(const net::Endpoint& endpoint, const std::string& bucket); + S3BucketName(const net::Endpoint& endpoint, std::string bucket); auto makeObject(const std::string& object) const -> std::unique_ptr; @@ -46,7 +52,7 @@ class S3BucketName: public S3Name { void ensureDestroyed(); - /// @todo: return S3 object iterator but first add prefix + /// @todo return S3 object iterator but first add prefix auto listObjects() const -> std::vector; auto asString() const -> std::string override; diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index 2a269ef85..894dc2a96 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -15,41 +15,40 @@ #include "eckit/io/s3/S3Client.h" -#include "S3Client.h" -#include "eckit/io/s3/S3Exception.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/s3/S3Config.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/log/CodeLocation.h" + +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Client::S3Client(const S3Config& config): config_(config) { } - -S3Client::~S3Client() = default; +S3Client::S3Client(S3Config config) : config_ {std::move(config)} { } //---------------------------------------------------------------------------------------------------------------------- -void S3Client::print(std::ostream& out) const { - out << "S3Client[config=" << config_ << "]"; -} +auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { -//---------------------------------------------------------------------------------------------------------------------- + if (config.backend == S3Backend::AWS) { return std::make_unique(config); } -auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr { - if (config.type == S3Types::AWS) { return std::make_shared(config); } - throw S3SeriousBug("Unkown S3 client type!", Here()); + throw UserError("Unsupported S3 backend! Supported backend = AWS ", Here()); } //---------------------------------------------------------------------------------------------------------------------- -auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - if (config.type == S3Types::AWS) { return std::make_unique(config); } - throw S3SeriousBug("Unkown S3 client type!", Here()); +void S3Client::print(std::ostream& out) const { + out << "S3Client[config=" << config_ << "]"; } -auto S3Client::endpoint() const -> const net::Endpoint& { - return config_.endpoint; +std::ostream& operator<<(std::ostream& out, const S3Client& client) { + client.print(out); + return out; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index d42e3792a..1d00c45de 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -20,9 +20,11 @@ #pragma once #include "eckit/io/s3/S3Config.h" -#include "eckit/memory/NonCopyable.h" +#include +#include #include +#include #include namespace eckit { @@ -31,15 +33,19 @@ class S3Context; //---------------------------------------------------------------------------------------------------------------------- -class S3Client: private NonCopyable { +class S3Client { public: // methods - virtual ~S3Client(); - - static auto makeShared(const S3Config& config) -> std::shared_ptr; + S3Client(const S3Client&) = delete; + S3Client& operator=(const S3Client&) = delete; + S3Client(S3Client&&) = default; + S3Client& operator=(S3Client&&) = default; + virtual ~S3Client() = default; static auto makeUnique(const S3Config& config) -> std::unique_ptr; - virtual auto endpoint() const -> const net::Endpoint&; + static auto makeShared(const S3Config& config) -> std::shared_ptr { return makeUnique(config); } + + auto config() const -> const S3Config& { return config_; } virtual void createBucket(const std::string& bucket) const = 0; @@ -51,11 +57,16 @@ class S3Client: private NonCopyable { virtual auto listBuckets() const -> std::vector = 0; - virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer, - uint64_t length) const -> long long = 0; + virtual auto putObject(const std::string& bucket, + const std::string& object, + const void* buffer, + uint64_t length) const -> long long = 0; - virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset, - uint64_t length) const -> long long = 0; + virtual auto getObject(const std::string& bucket, + const std::string& object, + void* buffer, + uint64_t offset, + uint64_t length) const -> long long = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; @@ -67,16 +78,15 @@ class S3Client: private NonCopyable { virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0; - friend std::ostream& operator<<(std::ostream& out, const S3Client& client) { - client.print(out); - return out; - } - protected: // methods - explicit S3Client(const S3Config& config); + S3Client(); + + explicit S3Client(S3Config config); virtual void print(std::ostream& out) const; + friend std::ostream& operator<<(std::ostream& out, const S3Client& client); + private: // members S3Config config_; }; diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index 8eedb780d..d857684f6 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -15,35 +15,112 @@ #include "eckit/io/s3/S3Config.h" +#include "eckit/config/LibEcKit.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/container/DenseSet.h" +#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include +#include +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Config::S3Config(std::string region, const URI& uri): region(std::move(region)), endpoint(uri) { } +namespace { + +const std::string defaultConfigFile = "~/.config/eckit/S3Config.yaml"; + +S3Config fromYAML(const LocalConfiguration& config) { + const net::Endpoint endpoint {config.getString("endpoint")}; + + const auto region = config.getString("region", s3DefaultRegion); + + S3Config s3config(endpoint, region); + + if (config.has("backend")) { + S3Backend backend {S3Backend::AWS}; // Default to AWS + std::string backendStr = config.getString("backend"); + if (backendStr == "AWS") { + backend = S3Backend::AWS; + } else if (backendStr == "REST") { + backend = S3Backend::REST; + } else if (backendStr == "MinIO") { + backend = S3Backend::MINIO; + } else { + throw UserError("Invalid backend: " + backendStr, Here()); + } + s3config.backend = backend; + } + + if (config.has("secure")) { s3config.secure = config.getBool("secure"); } + + return s3config; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Config::fromFile(std::string path) -> std::vector { -S3Config::S3Config(std::string region, const std::string& hostname, const int port): - region(std::move(region)), endpoint(hostname, port) { } + if (path.empty()) { path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", defaultConfigFile); } -S3Config::S3Config(const URI& uri): endpoint(uri) { } + PathName configFile(path); -S3Config::S3Config(const net::Endpoint& e): endpoint(e) { } + if (!configFile.exists()) { + Log::debug() << "S3 configuration file does not exist: " << configFile << std::endl; + return {}; + } + + const auto servers = YAMLConfiguration(configFile).getSubConfigurations("servers"); + + std::vector result; + result.reserve(servers.size()); + for (const auto& server : servers) { result.emplace_back(fromYAML(server)); } + + return result; +} + +S3Config::S3Config(const net::Endpoint& endpoint, std::string region) + : endpoint {endpoint}, region {std::move(region)} { } + +S3Config::S3Config(const std::string& host, const uint16_t port, std::string region) + : endpoint {host, port}, region {std::move(region)} { } + +S3Config::S3Config(const URI& uri) : S3Config(uri.host(), uri.port()) { } //---------------------------------------------------------------------------------------------------------------------- void S3Config::print(std::ostream& out) const { - out << "S3Config[type="; - if (type == S3Types::AWS) { + out << "S3Config[endpoint=" << endpoint << ",region=" << region << ",backend="; + if (backend == S3Backend::AWS) { out << "AWS"; - } else if (type == S3Types::MINIO) { - out << "MinIO"; - } else if (type == S3Types::REST) { + } else if (backend == S3Backend::REST) { out << "REST"; - } else { - out << "NONE"; + } else if (backend == S3Backend::MINIO) { + out << "MinIO"; } - out << ",region=" << region << ",endpoint=" << endpoint << "]"; + out << ",secure=" << secure << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3Config& config) { + config.print(out); + return out; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 5b5b981b9..dce4f0fc0 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -21,33 +21,69 @@ #include "eckit/net/Endpoint.h" +#include +#include #include +#include namespace eckit { -class URI; +enum class S3Backend : std::uint8_t { AWS, REST, MINIO }; -enum class S3Types { NONE, AWS, MINIO, REST }; +constexpr auto s3DefaultHost = "127.0.0.1"; +constexpr uint16_t s3DefaultPort = 443; +constexpr auto s3DefaultRegion = "default"; //---------------------------------------------------------------------------------------------------------------------- +/// @brief S3 configurations for a given endpoint +/// +/// @example Example YAML S3 configuration file: +/// +/// ECKIT_S3_CONFIG_FILE = ~/.config/eckit/S3Config.yaml +/// +/// --- +/// servers: +/// - endpoint: "127.0.0.1:9000" +/// region: "default" # (default) +/// secure: true # (default) +/// backend: "AWS" # (default) +/// +/// - endpoint: "minio:9000" +/// region: "eu-central-1" +/// +/// - endpoint: "https://localhost:9000" +/// region: "eu-central-1" +/// +/// # region is inferred from the endpoint +/// - endpoint: "https://eu-central-1.ecmwf.int:9000" +/// struct S3Config { - /// @todo region is part of hostname (s3express-control.region_code.amazonaws.com/bucket-name) - S3Config(std::string region, const URI& uri); - S3Config(std::string region, const std::string& hostname, int port); - S3Config(const URI& uri); - S3Config(const net::Endpoint&); - - friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { - config.print(out); - return out; + + static auto fromFile(std::string path) -> std::vector; + + S3Config() = default; + + explicit S3Config(const net::Endpoint& endpoint, std::string region = s3DefaultRegion); + + explicit S3Config(const std::string& host, uint16_t port, std::string region = s3DefaultRegion); + + explicit S3Config(const URI& uri); + + bool operator==(const S3Config& other) const { + return backend == other.backend && endpoint == other.endpoint && region == other.region; } + bool operator!=(const S3Config& other) const { return !(*this == other); } + void print(std::ostream& out) const; - S3Types type {S3Types::AWS}; - std::string region {"eu-central-1"}; - net::Endpoint endpoint {"127.0.0.1", -1}; + friend std::ostream& operator<<(std::ostream& out, const S3Config& config); + + net::Endpoint endpoint {s3DefaultHost, s3DefaultPort}; + std::string region {s3DefaultRegion}; + S3Backend backend {S3Backend::AWS}; + bool secure {true}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Credential.cc b/src/eckit/io/s3/S3Credential.cc new file mode 100644 index 000000000..cf33ac377 --- /dev/null +++ b/src/eckit/io/s3/S3Credential.cc @@ -0,0 +1,72 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Credential.h" + +#include "eckit/config/Configuration.h" +#include "eckit/config/LibEcKit.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/log/Log.h" + +#include +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +const std::string defaultCredFile = "~/.config/eckit/S3Credentials.yaml"; + +S3Credential fromYAML(const LocalConfiguration& config) { + const auto endpoint = config.getString("endpoint"); + const auto keyID = config.getString("accessKeyID"); + const auto secret = config.getString("secretKey"); + return {endpoint, keyID, secret}; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Credential::fromFile(std::string path) -> std::vector { + + if (path.empty()) { path = Resource("s3CredentialsFile;$ECKIT_S3_CREDENTIALS_FILE", defaultCredFile); } + + PathName credFile(path); + + if (!credFile.exists()) { + Log::debug() << "S3 credentials file does not exist: " << credFile << std::endl; + return {}; + } + + std::vector result; + + const auto creds = YAMLConfiguration(credFile).getSubConfigurations("credentials"); + result.reserve(creds.size()); + for (const auto& cred : creds) { result.emplace_back(fromYAML(cred)); } + + return result; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Credential.h b/src/eckit/io/s3/S3Credential.h index 79eb6f975..4cb9c594b 100644 --- a/src/eckit/io/s3/S3Credential.h +++ b/src/eckit/io/s3/S3Credential.h @@ -19,16 +19,48 @@ #pragma once +#include "eckit/net/Endpoint.h" + #include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- +/// @brief S3 credential information for a given endpoint +/// +/// @example Example YAML S3 credential file: +/// +/// ECKIT_S3_CREDENTIALS_FILE = ~/.config/eckit/S3Credentials.yaml +/// +/// --- +/// credentials: +/// - endpoint: '127.0.0.1:9000' +/// accessKeyID: 'minio' +/// secretKey: 'minio1234' +/// +/// - endpoint: 'minio:9000' +/// accessKeyID: 'minio' +/// secretKey: 'minio1234' +/// +/// - endpoint: 'localhost:9000' +/// accessKeyID: 'asd2' +/// secretKey: 'asd2' +/// struct S3Credential { - std::string endpoint {"127.0.0.1"}; - std::string keyID; - std::string secret; + + static auto fromFile(std::string path) -> std::vector; + + auto operator==(const S3Credential& other) const -> bool { + return endpoint == other.endpoint && keyID == other.keyID && secret == other.secret; + } + + auto operator!=(const S3Credential& other) const -> bool { return !(*this == other); } + + net::Endpoint endpoint; + std::string keyID; + std::string secret; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index 7ec075753..90d04798e 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -15,8 +15,12 @@ #include "eckit/io/s3/S3Exception.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/log/CodeLocation.h" + +#include #include -#include +#include //---------------------------------------------------------------------------------------------------------------------- @@ -34,20 +38,24 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc): SeriousBug("[S3 Error] " + msg, loc) { } +S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc) + : SeriousBug("[S3 Serious Bug] " + msg, loc) { } -S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc): - S3SeriousBug(addCode(msg, code), loc) { } +S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc) + : S3SeriousBug(addCode(msg, code), loc) { } //---------------------------------------------------------------------------------------------------------------------- -S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc): Exception("[S3 Exception] " + msg, loc) { } +S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc) : Exception("[S3 Exception] " + msg, loc) { } -S3BucketNotEmpty::S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } +S3BucketNotEmpty::S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Not Empty] " + msg, loc) { } -S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } +S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Not Found] " + msg, loc) { } -S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } +S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Already Exists] " + msg, loc) { } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index e49866710..173409ab7 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -15,46 +15,49 @@ /// @file S3Exception.h /// @author Metin Cakircali -/// @author Nicolau Manubens /// @date Jan 2024 #pragma once #include "eckit/exception/Exceptions.h" +#include "eckit/log/CodeLocation.h" +#include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3SeriousBug: public SeriousBug { +class S3SeriousBug : public SeriousBug { public: S3SeriousBug(const std::string& msg, const CodeLocation& loc); S3SeriousBug(const std::string& msg, int code, const CodeLocation& loc); - S3SeriousBug(const std::ostringstream& msg, const CodeLocation& loc): S3SeriousBug(msg.str(), loc) { } - S3SeriousBug(const std::ostringstream& msg, int code, const CodeLocation& loc): - S3SeriousBug(msg.str(), code, loc) { } + + S3SeriousBug(const std::ostringstream& msg, const CodeLocation& loc) : S3SeriousBug(msg.str(), loc) { } + + S3SeriousBug(const std::ostringstream& msg, int code, const CodeLocation& loc) + : S3SeriousBug(msg.str(), code, loc) { } }; //---------------------------------------------------------------------------------------------------------------------- -class S3Exception: public Exception { +class S3Exception : public Exception { public: S3Exception(const std::string& msg, const CodeLocation& loc); }; -class S3BucketNotEmpty: public S3Exception { +class S3BucketNotEmpty : public S3Exception { public: S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc); }; -class S3EntityNotFound: public S3Exception { +class S3EntityNotFound : public S3Exception { public: S3EntityNotFound(const std::string& msg, const CodeLocation& loc); }; -class S3EntityAlreadyExists: public S3Exception { +class S3EntityAlreadyExists : public S3Exception { public: S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc); }; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index aa6b4811e..6ffeafaa2 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -15,28 +15,35 @@ #include "eckit/io/s3/S3Name.h" +#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/utils/Tokenizer.h" +#include +#include +#include +#include +#include + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const URI& uri): endpoint_(uri), name_(uri.name()) { - ASSERT(uri.scheme() == "s3"); +S3Name::S3Name(const URI& uri) : endpoint_ {uri}, name_ {uri.name()} { + /// @todo is "s3://endpoint/bucket/object" a valid URI ? + ASSERT(uri.scheme() == type); } -S3Name::S3Name(const net::Endpoint& endpoint, const std::string& name): endpoint_(endpoint), name_(name) { } +S3Name::S3Name(const net::Endpoint& endpoint, std::string name) : endpoint_ {endpoint}, name_ {std::move(name)} { } S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- auto S3Name::uri() const -> URI { - return "s3://" + asString(); + return {type, asString()}; } auto S3Name::asString() const -> std::string { @@ -53,9 +60,16 @@ auto S3Name::parseName() const -> std::vector { return Tokenizer("/").tokenize(name_); } -auto S3Name::client() const -> std::shared_ptr { - /// @todo - return S3Session::instance().getClient({endpoint_}); +auto S3Name::client() const -> S3Client& { + if (!client_) { client_ = S3Session::instance().getClient(endpoint_); } + return *client_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3Name& name) { + name.print(out); + return out; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index c13a82d0a..7dc3ccff2 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -22,6 +22,7 @@ #include "eckit/net/Endpoint.h" +#include #include #include #include @@ -33,13 +34,26 @@ class S3Client; //---------------------------------------------------------------------------------------------------------------------- class S3Name { +public: // types + static constexpr auto type = "s3"; + public: // methods explicit S3Name(const URI& uri); - S3Name(const net::Endpoint& endpoint, const std::string& name); + S3Name(const net::Endpoint& endpoint, std::string name); + + // rules + + S3Name(const S3Name&) = default; + S3Name& operator=(const S3Name&) = default; + + S3Name(S3Name&&) = delete; + S3Name& operator=(S3Name&&) = delete; virtual ~S3Name(); + // accessors + auto uri() const -> URI; auto endpoint() const -> const net::Endpoint& { return endpoint_; } @@ -48,22 +62,21 @@ class S3Name { virtual auto asString() const -> std::string; - friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { - name.print(out); - return out; - } - protected: // methods virtual void print(std::ostream& out) const; + friend std::ostream& operator<<(std::ostream& out, const S3Name& name); + [[nodiscard]] auto parseName() const -> std::vector; - auto client() const -> std::shared_ptr; + auto client() const -> S3Client&; private: // members - const net::Endpoint endpoint_; - const std::string name_; + net::Endpoint endpoint_; + std::string name_; + + mutable std::shared_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index a859f82a9..27b89a6e2 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -15,23 +15,28 @@ #include "eckit/io/s3/S3ObjectName.h" -#include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" -#include "eckit/utils/Tokenizer.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3ObjectName::S3ObjectName(const URI& uri): S3Name(uri) { +S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri) { parse(); } -S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object): - S3Name(endpoint, "/"), bucket_(bucket), object_(object) { } +S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object) + : S3Name(endpoint, "/"), bucket_ {std::move(bucket)}, object_ {std::move(object)} { } //---------------------------------------------------------------------------------------------------------------------- @@ -54,27 +59,27 @@ auto S3ObjectName::asString() const -> std::string { } auto S3ObjectName::size() const -> long long { - return client()->objectSize(bucket_, object_); + return client().objectSize(bucket_, object_); } auto S3ObjectName::exists() const -> bool { - return client()->objectExists(bucket_, object_); + return client().objectExists(bucket_, object_); } auto S3ObjectName::bucketExists() const -> bool { - return client()->bucketExists(bucket_); + return client().bucketExists(bucket_); } void S3ObjectName::remove() { - client()->deleteObject(bucket_, object_); + client().deleteObject(bucket_, object_); } auto S3ObjectName::put(const void* buffer, const long length) const -> long long { - return client()->putObject(bucket_, object_, buffer, length); + return client().putObject(bucket_, object_, buffer, length); } auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long { - return client()->getObject(bucket_, object_, buffer, offset, length); + return client().getObject(bucket_, object_, buffer, offset, length); } auto S3ObjectName::dataHandle() -> DataHandle* { diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 919289162..447fab093 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -20,6 +20,10 @@ #pragma once #include "eckit/io/s3/S3Name.h" +#include "eckit/net/Endpoint.h" + +#include +#include namespace eckit { @@ -28,11 +32,11 @@ class DataHandle; //---------------------------------------------------------------------------------------------------------------------- -class S3ObjectName: public S3Name { +class S3ObjectName : public S3Name { public: // methods explicit S3ObjectName(const URI& uri); - S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object); + S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object); auto size() const -> long long; diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 270b0411c..9a899583f 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -15,36 +15,30 @@ #include "eckit/io/s3/S3Session.h" -#include "eckit/filesystem/LocalPathName.h" +#include "eckit/config/LibEcKit.h" #include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" #include "eckit/net/Endpoint.h" -#include "eckit/parser/YAMLParser.h" +#include "eckit/runtime/Main.h" -#include +#include +#include +#include namespace eckit { namespace { -/// @brief Functor for S3Context type -struct IsClientEndpoint { - IsClientEndpoint(const net::Endpoint& endpoint): endpoint_(endpoint) { } - - bool operator()(const std::shared_ptr& client) const { return client->endpoint() == endpoint_; } - -private: +/// @brief Predicate to find a client or credential by its endpoint +struct EndpointMatch { const net::Endpoint& endpoint_; -}; - -/// @brief Functor for S3Credential endpoint -struct IsCredentialEndpoint { - IsCredentialEndpoint(const std::string& endpoint): endpoint_(endpoint) { } bool operator()(const std::shared_ptr& cred) const { return cred->endpoint == endpoint_; } -private: - const std::string& endpoint_; + bool operator()(const std::shared_ptr& client) const { return client->config().endpoint == endpoint_; } }; } // namespace @@ -52,86 +46,101 @@ struct IsCredentialEndpoint { //---------------------------------------------------------------------------------------------------------------------- S3Session& S3Session::instance() { - thread_local S3Session session; + static S3Session session; return session; } //---------------------------------------------------------------------------------------------------------------------- -S3Session::S3Session() = default; +S3Session::S3Session() { + if (!Main::ready()) { + // inform when called before Main::initialise + Log::debug() << "Skipping initialization of S3Session instance.\n"; + return; + } + loadClients(); + loadCredentials(); +} S3Session::~S3Session() = default; //---------------------------------------------------------------------------------------------------------------------- void S3Session::clear() { - client_.clear(); + Log::debug() << "Clearing S3 clients and credentials.\n"; + clients_.clear(); credentials_.clear(); } //---------------------------------------------------------------------------------------------------------------------- // CLIENT -auto S3Session::getClient(const S3Config& config) -> std::shared_ptr { - // return if found - if (auto client = findClient(config.endpoint)) { return client; } +void S3Session::loadClients(const std::string& path) { + const auto configs = S3Config::fromFile(path); + for (const auto& config : configs) { addClient(config); } +} - // not found - auto client = S3Client::makeShared(config); - client_.push_back(client); +auto S3Session::findClient(const net::Endpoint& endpoint) const -> std::shared_ptr { - return client; -} + auto client = std::find_if(clients_.begin(), clients_.end(), EndpointMatch {endpoint}); + + if (client != clients_.end()) { return *client; } -auto S3Session::findClient(const net::Endpoint& endpoint) -> std::shared_ptr { - // search by type - const auto client = std::find_if(client_.begin(), client_.end(), IsClientEndpoint(endpoint)); - // found - if (client != client_.end()) { return *client; } // not found return {}; } +auto S3Session::getClient(const net::Endpoint& endpoint) const -> std::shared_ptr { + + if (auto client = findClient(endpoint)) { return client; } + + throw S3EntityNotFound("Could not find the client for " + std::string(endpoint), Here()); +} + +auto S3Session::addClient(const S3Config& config) -> std::shared_ptr { + if (auto client = findClient(config.endpoint)) { return client; } + // not found, add new item + return clients_.emplace_back(S3Client::makeShared(config)); +} + void S3Session::removeClient(const net::Endpoint& endpoint) { - client_.remove_if(IsClientEndpoint(endpoint)); + clients_.remove_if(EndpointMatch {endpoint}); } //---------------------------------------------------------------------------------------------------------------------- // CREDENTIALS -void S3Session::readCredentials(std::string path) { - if (path.empty()) { - path = Resource("$ECKIT_S3_CREDENTIALS_FILE", "~/.config/eckit/s3credentials.yaml"); - } - - const PathName credFile(path, true); - if (credFile.exists()) { - const ValueList creds = YAMLParser::decodeFile(credFile); - for (auto&& cred : creds) { addCredentials({cred["endpoint"], cred["accessKeyID"], cred["secretKey"]}); } - } else { - Log::warning() << ReadError(credFile).what() << std::endl; - } +void S3Session::loadCredentials(const std::string& path) { + const auto creds = S3Credential::fromFile(path); + for (const auto& cred : creds) { addCredential(cred); } } -auto S3Session::getCredentials(const net::Endpoint& endpoint) const -> std::shared_ptr { - // search by endpoint - const auto cred = std::find_if(credentials_.begin(), credentials_.end(), IsCredentialEndpoint(endpoint)); - // found +auto S3Session::findCredential(const net::Endpoint& endpoint) const -> std::shared_ptr { + + auto cred = std::find_if(credentials_.begin(), credentials_.end(), EndpointMatch {endpoint}); + if (cred != credentials_.end()) { return *cred; } + // not found return {}; } -void S3Session::addCredentials(const S3Credential& credential) { - // check if already exists - if (getCredentials(credential.endpoint)) { return; } - // add new item - auto cred = std::make_shared(credential); - credentials_.emplace_back(cred); +auto S3Session::getCredential(const net::Endpoint& endpoint) const -> std::shared_ptr { + + if (auto cred = findCredential(endpoint)) { return cred; } + + throw S3EntityNotFound("Could not find the credential for " + std::string(endpoint), Here()); +} + +auto S3Session::addCredential(const S3Credential& credential) -> std::shared_ptr { + // don't add duplicate credentials + if (auto cred = findCredential(credential.endpoint)) { return cred; } + // not found: add new item + return credentials_.emplace_back(std::make_shared(credential)); } -void S3Session::removeCredentials(const std::string& endpoint) { - credentials_.remove_if(IsCredentialEndpoint(endpoint)); +void S3Session::removeCredential(const net::Endpoint& endpoint) { + credentials_.remove_if(EndpointMatch {endpoint}); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 6542cf1ef..50a042b3a 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -19,14 +19,14 @@ #pragma once -#include "eckit/io/s3/S3Credential.h" - #include #include +#include namespace eckit { class S3Client; +struct S3Credential; struct S3Config; namespace net { @@ -35,7 +35,7 @@ class Endpoint; //---------------------------------------------------------------------------------------------------------------------- -class S3Session { +class S3Session final { public: // methods S3Session(const S3Session&) = delete; S3Session& operator=(const S3Session&) = delete; @@ -46,30 +46,50 @@ class S3Session { void clear(); + // clients + + void loadClients(const std::string& path = ""); + + /// @brief Get an S3 client for the given configuration + /// @param config S3 configuration + /// @return S3 client + /// @throws S3EntityNotFound if the client does not exist [[nodiscard]] - auto getCredentials(const net::Endpoint& endpoint) const -> std::shared_ptr; + auto getClient(const net::Endpoint& endpoint) const -> std::shared_ptr; - void readCredentials(std::string path = ""); + /// @brief Add an S3 client for the given configuration + /// @param config S3 configuration + /// @return S3 client + auto addClient(const S3Config& config) -> std::shared_ptr; + + void removeClient(const net::Endpoint& endpoint); - void addCredentials(const S3Credential& credential); + // credentials - void removeCredentials(const std::string& endpoint); + void loadCredentials(const std::string& path = ""); + /// @brief Get an S3 credential for the given configuration + /// @param endpoint S3 endpoint + /// @return S3 credential + /// @throws S3EntityNotFound if the credential does not exist [[nodiscard]] - auto getClient(const S3Config& config) -> std::shared_ptr; + auto getCredential(const net::Endpoint& endpoint) const -> std::shared_ptr; - void removeClient(const net::Endpoint& endpoint); + auto addCredential(const S3Credential& credential) -> std::shared_ptr; + + void removeCredential(const net::Endpoint& endpoint); private: // methods S3Session(); ~S3Session(); - [[nodiscard]] - auto findClient(const net::Endpoint& endpoint) -> std::shared_ptr; + auto findClient(const net::Endpoint& endpoint) const -> std::shared_ptr; + + auto findCredential(const net::Endpoint& endpoint) const -> std::shared_ptr; private: // members - std::list> client_; + std::list> clients_; std::list> credentials_; }; diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc index ef96dfdc4..c8749f56a 100644 --- a/src/eckit/io/s3/S3URIManager.cc +++ b/src/eckit/io/s3/S3URIManager.cc @@ -15,17 +15,20 @@ #include "eckit/io/s3/S3URIManager.h" +#include "eckit/filesystem/URIManager.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" #include "eckit/io/s3/S3ObjectName.h" +#include + namespace eckit { static S3URIManager manager_s3("s3"); //---------------------------------------------------------------------------------------------------------------------- -S3URIManager::S3URIManager(const std::string& name): URIManager(name) { } - -S3URIManager::~S3URIManager() = default; +S3URIManager::S3URIManager(const std::string& name) : URIManager(name) { } bool S3URIManager::exists(const URI& uri) { return S3ObjectName(uri).exists(); @@ -39,7 +42,7 @@ DataHandle* S3URIManager::newReadHandle(const URI& uri) { return S3ObjectName(uri).dataHandle(); } -DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList&, const LengthList&) { +DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList& /*offsets*/, const LengthList& /*lengths*/) { return S3ObjectName(uri).dataHandle(); } diff --git a/src/eckit/io/s3/S3URIManager.h b/src/eckit/io/s3/S3URIManager.h index 7a5262e4e..9b3dc5e1a 100644 --- a/src/eckit/io/s3/S3URIManager.h +++ b/src/eckit/io/s3/S3URIManager.h @@ -22,24 +22,24 @@ #include "eckit/filesystem/URIManager.h" +#include + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3URIManager: public URIManager { +class S3URIManager : public URIManager { public: // methods - S3URIManager(const std::string& name); - - ~S3URIManager() override; + explicit S3URIManager(const std::string& name); bool authority() override { return true; } private: // methods - bool exists(const URI&) override; + bool exists(const URI& uri) override; - DataHandle* newWriteHandle(const URI&) override; - DataHandle* newReadHandle(const URI&) override; - DataHandle* newReadHandle(const URI&, const OffsetList&, const LengthList&) override; + DataHandle* newWriteHandle(const URI& uri) override; + DataHandle* newReadHandle(const URI& uri) override; + DataHandle* newReadHandle(const URI& uri, const OffsetList& offsets, const LengthList& lengths) override; std::string asString(const URI& uri) const override; }; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index ee0baf80b..b1d4d9906 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -16,13 +16,26 @@ #include "eckit/io/s3/aws/S3ClientAWS.h" #include "eckit/config/LibEcKit.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ContextAWS.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/IPAddress.h" #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include #include #include @@ -32,10 +45,15 @@ #include #include #include +#include #include +#include #include #include +#include +#include +#include //---------------------------------------------------------------------------------------------------------------------- @@ -47,10 +65,10 @@ inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Erro return oss.str(); } -class BufferIOStream: public Aws::IOStream { +class BufferIOStream : public Aws::IOStream { public: - BufferIOStream(void* buffer, const uint64_t length): - Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), length)) { + BufferIOStream(void* buffer, const uint64_t length) + : Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), length)) { } ~BufferIOStream() override { delete rdbuf(); } @@ -62,52 +80,53 @@ class BufferIOStream: public Aws::IOStream { namespace eckit { -const auto ALLOC_TAG = "S3ClientAWS"; +const auto allocTag = "S3ClientAWS"; -S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(config), ctx_(S3ContextAWS::instance()) { - configure(config); -} - -S3ClientAWS::~S3ClientAWS() = default; +S3ClientAWS::S3ClientAWS(const S3Config& config) : S3Client(config), ctx_ {S3ContextAWS::instance()} { } //---------------------------------------------------------------------------------------------------------------------- -void S3ClientAWS::configure(const S3Config& config) { - LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client..." << std::endl; +void S3ClientAWS::configure() const { + + LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client... "; Aws::Client::ClientConfigurationInitValues initVal; initVal.shouldDisableIMDS = true; - Aws::Client::ClientConfiguration configuration(initVal); + Aws::Client::ClientConfiguration configAWS(initVal); // we are not an ec2 instance - configuration.disableIMDS = true; - configuration.disableImdsV1 = true; + configAWS.disableIMDS = true; + configAWS.disableImdsV1 = true; // setup region - if (!config.region.empty()) { configuration.region = config.region; } + if (!config().region.empty()) { configAWS.region = config().region; } // configuration.scheme = Aws::Http::Scheme::HTTPS; - configuration.verifySSL = false; + configAWS.verifySSL = false; // setup endpoint - if (!config.endpoint.host().empty()) { configuration.endpointOverride = config.endpoint.host(); } - if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } - - if (auto cred = S3Session::instance().getCredentials(config.endpoint)) { - // credentials provider - auto cProvider = Aws::MakeShared(ALLOC_TAG, cred->keyID, cred->secret); - // endpoint provider - auto eProvider = Aws::MakeShared(ALLOC_TAG); - // client - client_ = std::make_unique(cProvider, eProvider, configuration); - } else { - throw S3SeriousBug("No credentials found!", Here()); - } + if (config().endpoint.host().empty()) { throw UserError("Empty endpoint hostname in configuration!", Here()); } + + configAWS.endpointOverride = net::IPAddress::hostAddress(config().endpoint.host()).asString(); + + ASSERT(config().endpoint.port() > 0); + configAWS.endpointOverride += ":" + std::to_string(config().endpoint.port()); + + LOG_DEBUG_LIB(LibEcKit) << "endpoint=" << configAWS.endpointOverride << std::endl; + + const auto cred = S3Session::instance().getCredential(config().endpoint); + // credentials provider + auto cProvider = Aws::MakeShared(allocTag, cred->keyID, cred->secret); + // endpoint provider + auto eProvider = Aws::MakeShared(allocTag); + // client + client_ = std::make_unique(cProvider, eProvider, configAWS); } -auto S3ClientAWS::getClient() const -> Aws::S3::S3Client& { - if (client_) { return *client_; } - throw S3SeriousBug("Invalid client!", Here()); +auto S3ClientAWS::client() const -> Aws::S3::S3Client& { + if (!client_) { configure(); } + if (!client_) { throw S3SeriousBug("Invalid client!", Here()); } + return *client_; } //---------------------------------------------------------------------------------------------------------------------- @@ -117,7 +136,7 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { Aws::S3::Model::CreateBucketRequest request; request.SetBucket(bucket); - auto outcome = getClient().CreateBucket(request); + auto outcome = client().CreateBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -129,6 +148,8 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { throw S3SeriousBug(msg, Here()); } + /// @todo do we wait for the bucket to propagate? + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; } @@ -142,7 +163,7 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucket); - auto outcome = getClient().DeleteBucket(request); + auto outcome = client().DeleteBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -164,13 +185,13 @@ auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { request.SetBucket(bucket); - return getClient().HeadBucket(request).IsSuccess(); + return client().HeadBucket(request).IsSuccess(); } //---------------------------------------------------------------------------------------------------------------------- auto S3ClientAWS::listBuckets() const -> std::vector { - auto outcome = getClient().ListBuckets(); + auto outcome = client().ListBuckets(); if (outcome.IsSuccess()) { std::vector buckets; @@ -185,8 +206,10 @@ auto S3ClientAWS::listBuckets() const -> std::vector { //---------------------------------------------------------------------------------------------------------------------- // PUT OBJECT -auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const -> long long { +auto S3ClientAWS::putObject(const std::string& bucket, + const std::string& object, + const void* buffer, + const uint64_t length) const -> long long { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket); @@ -194,14 +217,14 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object // request.SetContentLength(length); if (buffer && length > 0) { - auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); + auto streamBuffer = Aws::MakeShared(allocTag, const_cast(buffer), length); request.SetBody(streamBuffer); } else { // empty object - request.SetBody(Aws::MakeShared(ALLOC_TAG)); + request.SetBody(Aws::MakeShared(allocTag)); } - auto outcome = getClient().PutObject(request); + auto outcome = client().PutObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; @@ -216,17 +239,20 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object //---------------------------------------------------------------------------------------------------------------------- // GET OBJECT -auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, +auto S3ClientAWS::getObject(const std::string& bucket, + const std::string& object, + void* buffer, + const uint64_t /*offset*/, const uint64_t length) const -> long long { Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(object); - request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); + request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(allocTag, buffer, length); }); /// @todo range and streambuf // request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); - auto outcome = getClient().GetObject(request); + auto outcome = client().GetObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; @@ -248,7 +274,7 @@ void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& obj request.SetBucket(bucket); request.SetKey(object); - auto outcome = getClient().DeleteObject(request); + auto outcome = client().DeleteObject(request); if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete object=" + object + " in bucket=" + bucket, outcome.GetError()); @@ -273,7 +299,7 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index fb9459f12..a567332d4 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -74,21 +74,11 @@ ecbuild_add_test( TARGET eckit_test_radoshandle INCLUDES ${RADOS_INCLUDE_DIRS} LIBS eckit ) -ecbuild_add_test( TARGET eckit_test_s3client - SOURCES test_s3client.cc - CONDITION HAVE_AWS_S3 - INCLUDES ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ) - -ecbuild_add_test( TARGET eckit_test_s3handle - SOURCES test_s3handle.cc - CONDITION HAVE_AWS_S3 - INCLUDES ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ) - ecbuild_add_test( TARGET eckit_rados-performance SOURCES rados-performance.cc CONDITION HAVE_EXTRA_TESTS AND HAVE_RADOS INCLUDES ${RADOS_INCLUDE_DIRS} TEST_DEPENDS get_eckit_io_test_data LIBS eckit ) + +add_subdirectory( s3 ) diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt new file mode 100644 index 000000000..4adca6508 --- /dev/null +++ b/tests/io/s3/CMakeLists.txt @@ -0,0 +1,23 @@ + +if( eckit_HAVE_AWS_S3 ) + + file( + COPY S3Config.yaml S3Credentials.yaml + DESTINATION ${CMAKE_CURRENT_BINARY_DIR} + ) + + ecbuild_add_test( + TARGET eckit_test_s3client + SOURCES test_s3client.cc + INCLUDES ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ${AWSSDK_LINK_LIBRARIES} + ) + + ecbuild_add_test( + TARGET eckit_test_s3handle + SOURCES test_s3handle.cc + INCLUDES ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ${AWSSDK_LINK_LIBRARIES} + ) + +endif( eckit_HAVE_AWS_S3 ) diff --git a/tests/io/s3/S3Config.yaml b/tests/io/s3/S3Config.yaml new file mode 100644 index 000000000..fd2026ec9 --- /dev/null +++ b/tests/io/s3/S3Config.yaml @@ -0,0 +1,15 @@ +--- +servers: + - endpoint: "127.0.0.1:9000" + region: "default" # (default) + secure: true # (default) + backend: "AWS" # (default) + + - endpoint: "minio:9000" + region: "eu-central-1" + + - endpoint: "localhost:9000" + region: "eu-central-1" + + # region is inferred from the endpoint + - endpoint: "eu-central-1.ecmwf.int:9000" diff --git a/tests/io/s3/S3Credentials.yaml b/tests/io/s3/S3Credentials.yaml new file mode 100644 index 000000000..1935a4815 --- /dev/null +++ b/tests/io/s3/S3Credentials.yaml @@ -0,0 +1,13 @@ +--- +credentials: + - endpoint: '127.0.0.1:9000' + accessKeyID: 'minio' + secretKey: 'minio1234' + + - endpoint: 'minio:9000' + accessKeyID: 'minio' + secretKey: 'minio1234' + + - endpoint: 'localhost:9000' + accessKeyID: 'asd2' + secretKey: 'asd2' diff --git a/tests/io/s3/test_s3client.cc b/tests/io/s3/test_s3client.cc new file mode 100644 index 000000000..09be236ff --- /dev/null +++ b/tests/io/s3/test_s3client.cc @@ -0,0 +1,218 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file test_s3client.cc +/// @author Metin Cakircali +/// @author Simon Smart +/// @date Jan 2024 + +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/net/Endpoint.h" +#include "eckit/testing/Test.h" + +#include +#include +#include + +using namespace eckit; +using namespace eckit::testing; + +// this test requires a working S3 endpoint and credentials. such as, a local docker container MinIO instance + +namespace eckit::test { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +const net::Endpoint TEST_ENDPOINT {"minio", 9000}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"}; + +bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +void cleanup() { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto* name : {"test-bucket-1", "test-bucket-2"}) { + if (client->bucketExists(name)) { + client->emptyBucket(name); + client->deleteBucket(name); + } + } +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 client: API") { + + const S3Config config {TEST_ENDPOINT, "eu-central-1"}; + const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"}; + + EXPECT(S3Session::instance().addClient(config)); + EXPECT(S3Session::instance().addCredential(cred)); + + EXPECT_NO_THROW(cleanup()); + + EXPECT_NO_THROW(S3Session::instance().removeClient(config.endpoint)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 client: read from file") { + + auto& session = S3Session::instance(); + + session.clear(); + + EXPECT_THROWS_AS(session.getClient({"127.0.0.1", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getClient({"minio", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getClient({"localhost", 9000}), S3EntityNotFound); + + session.loadClients("./S3Config.yaml"); + + EXPECT_NO_THROW(session.getClient({"127.0.0.1", 9000})); + EXPECT_NO_THROW(session.getClient({"minio", 9000})); + EXPECT_NO_THROW(session.getClient({"localhost", 9000})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 credentials: API") { + + const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"}; + + EXPECT(S3Session::instance().addCredential(cred)); + + EXPECT_NO_THROW(cleanup()); + + EXPECT_NO_THROW(S3Session::instance().removeCredential(cred.endpoint)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 credentials: read from file") { + + auto& session = S3Session::instance(); + + session.clear(); + + EXPECT_THROWS_AS(session.getCredential({"127.0.0.1", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getCredential({"minio", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getCredential({"localhost", 9000}), S3EntityNotFound); + + session.loadCredentials("./S3Credentials.yaml"); + + EXPECT_NO_THROW(session.getCredential({"127.0.0.1", 9000})); + EXPECT_NO_THROW(session.getCredential({"minio", 9000})); + EXPECT_NO_THROW(session.getCredential({"localhost", 9000})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 backends") { + S3Config cfgTmp(TEST_CONFIG); + + cfgTmp.backend = S3Backend::AWS; + EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); + + cfgTmp.backend = S3Backend::REST; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); + + cfgTmp.backend = S3Backend::MINIO; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); +} + +// CASE("wrong credentials") { +// ensureClean(); +// +// S3Config cfgTmp(cfg); +// cfgTmp.region = "no-region-random"; +// +// EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("failed-bucket")); +// } + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create s3 bucket in non-existing region") { + EXPECT_NO_THROW(cleanup()); + + // this test requires an S3 endpoint that sets it's region + // a MinIO instance with empty region will not throw an exception + + auto cfgTmp = TEST_CONFIG; + cfgTmp.region = "non-existing-region-random"; + + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("test-bucket-1")); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create s3 bucket") { + EXPECT_NO_THROW(cleanup()); + + auto client = S3Client::makeUnique(TEST_CONFIG); + + EXPECT_NO_THROW(client->createBucket("test-bucket-1")); + + EXPECT_THROWS(client->createBucket("test-bucket-1")); + + EXPECT_NO_THROW(client->createBucket("test-bucket-2")); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("list s3 buckets") { + EXPECT_NO_THROW(cleanup()); + + auto client = S3Client::makeUnique(TEST_CONFIG); + EXPECT_NO_THROW(client->createBucket("test-bucket-1")); + EXPECT_NO_THROW(client->createBucket("test-bucket-2")); + + { + const auto buckets = client->listBuckets(); + + EXPECT(findString(buckets, "test-bucket-1")); + EXPECT(findString(buckets, "test-bucket-2")); + + EXPECT_NO_THROW(client->deleteBucket("test-bucket-1")); + EXPECT_NO_THROW(client->deleteBucket("test-bucket-2")); + } + + { + const auto buckets = client->listBuckets(); + EXPECT_NOT(findString(buckets, "test-bucket-1")); + EXPECT_NOT(findString(buckets, "test-bucket-2")); + } + + EXPECT_THROWS(client->deleteBucket("test-bucket-1")); + EXPECT_THROWS(client->deleteBucket("test-bucket-2")); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test + +int main(int argc, char** argv) { + int ret = -1; + + ret = run_tests(argc, argv); + + test::cleanup(); + + return ret; +} diff --git a/tests/io/test_s3handle.cc b/tests/io/s3/test_s3handle.cc similarity index 58% rename from tests/io/test_s3handle.cc rename to tests/io/s3/test_s3handle.cc index db5c84530..8c5fcba2a 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/s3/test_s3handle.cc @@ -19,28 +19,42 @@ #include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3BucketName.h" #include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Handle.h" +#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3Session.h" #include "eckit/log/Bytes.h" #include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" +#include + #include +#include +#include +#include #include +#include + +using namespace std::string_literals; -using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz"; -static const std::string TEST_BUCKET("eckit-s3handle-test-bucket"); -static const std::string TEST_OBJECT("eckit-s3handle-test-object"); +const net::Endpoint TEST_ENDPOINT {"minio", 9000}; +const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; +const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; -S3Config cfg("eu-central-1", "127.0.0.1", 9000); +const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"}; +const S3Credential TEST_CREDENTIAL {TEST_ENDPOINT, "minio", "minio1234"}; //---------------------------------------------------------------------------------------------------------------------- @@ -61,8 +75,8 @@ void writePerformance(S3BucketName& bucket, const int count) { << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; } -void ensureClean() { - auto client = S3Client::makeUnique(cfg); +void cleanup() { + auto client = S3Client::makeUnique(TEST_CONFIG); for (const auto& name : {TEST_BUCKET}) { if (client->bucketExists(name)) { client->emptyBucket(name); @@ -71,20 +85,30 @@ void ensureClean() { } } +} // namespace + //---------------------------------------------------------------------------------------------------------------------- -CASE("invalid bucket") { - EXPECT_THROWS(S3BucketName("http://127.0.0.1:9000/" + TEST_BUCKET)); - EXPECT_THROWS(S3BucketName("s3://127.0.0.1" + TEST_BUCKET)); - EXPECT_THROWS(S3BucketName("s3://127.0.0.1/" + TEST_BUCKET)); +CASE("initialize s3 session") { + + EXPECT(S3Session::instance().addCredential(TEST_CREDENTIAL)); + EXPECT(S3Session::instance().addClient(TEST_CONFIG)); + + EXPECT_NO_THROW(cleanup()); +} + +CASE("invalid s3 bucket") { + EXPECT_THROWS(S3BucketName(URI {"http://127.0.0.1:9000/" + TEST_BUCKET})); + EXPECT_THROWS(S3BucketName(URI {"s3://127.0.0.1" + TEST_BUCKET})); + EXPECT_THROWS(S3BucketName(URI {"s3://127.0.0.1/" + TEST_BUCKET})); } //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: no bucket") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -99,10 +123,12 @@ CASE("S3BucketName: no bucket") { EXPECT_NO_THROW(bucket.ensureDestroyed()); } +//---------------------------------------------------------------------------------------------------------------------- + CASE("S3BucketName: create bucket") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -115,26 +141,30 @@ CASE("S3BucketName: create bucket") { EXPECT_NO_THROW(bucket.destroy()); } +//---------------------------------------------------------------------------------------------------------------------- + CASE("S3BucketName: empty bucket") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); EXPECT(bucket.exists()); // LIST - EXPECT(bucket.listObjects().size() == 0); + EXPECT_EQUAL(bucket.listObjects().size(), 0); // DESTROY BUCKET EXPECT_NO_THROW(bucket.destroy()); } +//---------------------------------------------------------------------------------------------------------------------- + CASE("S3BucketName: bucket with object") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); @@ -143,7 +173,7 @@ CASE("S3BucketName: bucket with object") { EXPECT_NO_THROW(bucket.makeObject(TEST_OBJECT)->put(TEST_DATA.data(), TEST_DATA.size())); // LIST - EXPECT(bucket.listObjects().size() == 1); + EXPECT_EQUAL(bucket.listObjects().size(), 1); // DESTROY BUCKET EXPECT_THROWS(bucket.destroy()); @@ -152,22 +182,22 @@ CASE("S3BucketName: bucket with object") { //---------------------------------------------------------------------------------------------------------------------- -CASE("S3Handle operations") { - ensureClean(); +CASE("S3Handle: basic operations") { + EXPECT_NO_THROW(cleanup()); - S3Client::makeUnique(cfg)->createBucket(TEST_BUCKET); + S3Client::makeUnique(TEST_CONFIG)->createBucket(TEST_BUCKET); const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { S3ObjectName object(uri); std::unique_ptr handle(object.dataHandle()); EXPECT_NO_THROW(handle->openForWrite(length)); - EXPECT(handle->write(buffer, length) == length); + EXPECT_EQUAL(handle->write(buffer, length), length); EXPECT_NO_THROW(handle->close()); } @@ -180,7 +210,7 @@ CASE("S3Handle operations") { EXPECT_NO_THROW(handle->read(rbuf.data(), length)); - EXPECT(rbuf == TEST_DATA); + EXPECT_EQUAL(rbuf, TEST_DATA); EXPECT_NO_THROW(handle->close()); } @@ -191,17 +221,19 @@ CASE("S3Handle operations") { MemoryHandle memHandle(length); handle->saveInto(memHandle); - EXPECT(memHandle.size() == Length(length)); - EXPECT(::memcmp(memHandle.data(), buffer, length) == 0); + EXPECT_EQUAL(memHandle.size(), Length(length)); + EXPECT_EQUAL(::memcmp(memHandle.data(), buffer, length), 0); EXPECT_NO_THROW(handle->close()); } } -CASE("S3Handle::openForRead") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: openForRead") { + EXPECT_NO_THROW(cleanup()); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -221,10 +253,12 @@ CASE("S3Handle::openForRead") { EXPECT_NO_THROW(handle->openForRead()); } -CASE("S3Handle::openForWrite") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); +CASE("S3Handle: openForWrite") { + EXPECT_NO_THROW(cleanup()); + + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { // NO BUCKET std::unique_ptr handle(uri.newWriteHandle()); @@ -242,10 +276,12 @@ CASE("S3Handle::openForWrite") { } } -CASE("S3Handle::read") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: read") { + EXPECT_NO_THROW(cleanup()); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -257,37 +293,39 @@ CASE("S3Handle::read") { // OPEN EXPECT_NO_THROW(handle->openForRead()); - { /// @todo range based read - - // const auto length = TEST_DATA.size(); - - // std::string rbuf; - // rbuf.resize(length); - // auto len = handle->read(rbuf.data(), 10); - // std::cout << "========" << rbuf << std::endl; - // len = handle->read(rbuf.data(), length - 10); - // std::cout << "========" << rbuf << std::endl; - // EXPECT(rbuf == TEST_DATA); - } + /// @todo range based read + // { + // const auto length = TEST_DATA.size(); + // + // std::string rbuf; + // rbuf.resize(length); + // auto len = handle->read(rbuf.data(), 10); + // std::cout << "========" << rbuf << std::endl; + // len = handle->read(rbuf.data(), length - 10); + // std::cout << "========" << rbuf << std::endl; + // EXPECT_EQUAL(rbuf, TEST_DATA); + // } // CLOSE EXPECT_NO_THROW(handle->close()); } -CASE("performance: write 10 100 1000 objects") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 performance: write 1 10 100 objects") { + EXPECT_NO_THROW(cleanup()); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET); S3BucketName bucket(uri); EXPECT_NO_THROW(bucket.ensureCreated()); + writePerformance(bucket, 1); + writePerformance(bucket, 10); writePerformance(bucket, 100); - - writePerformance(bucket, 1000); } //---------------------------------------------------------------------------------------------------------------------- @@ -295,14 +333,13 @@ CASE("performance: write 10 100 1000 objects") { } // namespace eckit::test int main(int argc, char** argv) { - const S3Credential cred {"127.0.0.1:9000", "minio", "minio1234"}; - S3Session::instance().addCredentials(cred); - int ret = -1; - try { - ret = run_tests(argc, argv); - } - catch (...) { - } + + ret = run_tests(argc, argv); + + test::cleanup(); + return ret; } + +//---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc deleted file mode 100644 index aa94427ca..000000000 --- a/tests/io/test_s3client.cc +++ /dev/null @@ -1,135 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/// @file test_s3client.cc -/// @author Metin Cakircali -/// @author Simon Smart -/// @date Jan 2024 - -#include "eckit/config/LibEcKit.h" -#include "eckit/io/Buffer.h" -#include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Session.h" -#include "eckit/testing/Test.h" - -using namespace std; -using namespace eckit; -using namespace eckit::testing; - -namespace eckit::test { - -const S3Config cfg("eu-central-1", "127.0.0.1", 9000); - -//---------------------------------------------------------------------------------------------------------------------- - -bool findString(const std::vector& list, const std::string& item) { - return (std::find(list.begin(), list.end(), item) != list.end()); -} - -void ensureClean() { - - auto client = S3Client::makeUnique(cfg); - auto&& tmp = client->listBuckets(); - std::set buckets(tmp.begin(), tmp.end()); - - for (const std::string& name : {"test-bucket-1", "test-bucket-2"}) { - if (buckets.find(name) != buckets.end()) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } -} - -//---------------------------------------------------------------------------------------------------------------------- - -CASE("different types") { - S3Config cfgTmp(cfg); - - cfgTmp.type = S3Types::NONE; - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); - - cfgTmp.type = S3Types::AWS; - EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); - - cfgTmp.type = S3Types::REST; - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); -} - -// CASE("wrong credentials") { -// ensureClean(); -// -// S3Config cfgTmp(cfg); -// cfgTmp.region = "no-region-random"; -// -// EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("failed-bucket")); -// } - -CASE("create bucket in non-existing region") { - ensureClean(); - - S3Config cfgTmp(cfg); - cfgTmp.region = "non-existing-region-random"; - - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("test-bucket-1")); -} - -CASE("create bucket") { - ensureClean(); - - auto client = S3Client::makeUnique(cfg); - - EXPECT_NO_THROW(client->createBucket("test-bucket-1")); - - EXPECT_THROWS(client->createBucket("test-bucket-1")); - - EXPECT_NO_THROW(client->createBucket("test-bucket-2")); -} - -CASE("list buckets") { - ensureClean(); - - auto client = S3Client::makeUnique(cfg); - EXPECT_NO_THROW(client->createBucket("test-bucket-1")); - EXPECT_NO_THROW(client->createBucket("test-bucket-2")); - - { - const auto buckets = client->listBuckets(); - - EXPECT(findString(buckets, "test-bucket-1")); - EXPECT(findString(buckets, "test-bucket-2")); - - EXPECT_NO_THROW(client->deleteBucket("test-bucket-1")); - EXPECT_NO_THROW(client->deleteBucket("test-bucket-2")); - } - - { - const auto buckets = client->listBuckets(); - EXPECT_NOT(findString(buckets, "test-bucket-1")); - EXPECT_NOT(findString(buckets, "test-bucket-2")); - } - - EXPECT_THROWS(client->deleteBucket("test-bucket-1")); - EXPECT_THROWS(client->deleteBucket("test-bucket-2")); -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit::test - -int main(int argc, char** argv) { - const S3Credential cred {"127.0.0.1:9000", "minio", "minio1234"}; - S3Session::instance().addCredentials(cred); - - auto ret = run_tests(argc, argv); - try { - eckit::test::ensureClean(); - } catch (...) { } - return ret; -}