Skip to content

Commit

Permalink
feat(S3): add config and credentials from file
Browse files Browse the repository at this point in the history 
fix several minor issues
  • Loading branch information
@mcakircali
mcakircali committed Dec 17, 2024
1 parent 45b7135 commit 536785a
Show file tree
Hide file tree
Showing 29 changed files with 1,023 additions and 510 deletions.
57 changes: 29 additions & 28 deletions src/eckit/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,33 +283,34 @@ if(HAVE_RADOS)
)
endif()

if(HAVE_AWS_S3)
list( APPEND eckit_io_srcs
io/s3/aws/S3ClientAWS.cc
io/s3/aws/S3ClientAWS.h
io/s3/aws/S3ContextAWS.cc
io/s3/aws/S3ContextAWS.h
io/s3/S3BucketName.cc
io/s3/S3BucketName.h
io/s3/S3Client.cc
io/s3/S3Client.h
io/s3/S3Config.cc
io/s3/S3Config.h
io/s3/S3Credential.h
io/s3/S3Exception.cc
io/s3/S3Exception.h
io/s3/S3Handle.cc
io/s3/S3Handle.h
io/s3/S3Name.cc
io/s3/S3Name.h
io/s3/S3ObjectName.cc
io/s3/S3ObjectName.h
io/s3/S3Session.cc
io/s3/S3Session.h
io/s3/S3URIManager.cc
io/s3/S3URIManager.h
)
endif(HAVE_AWS_S3)
if( eckit_HAVE_AWS_S3 )
list( APPEND eckit_io_srcs
io/s3/aws/S3ClientAWS.cc
io/s3/aws/S3ClientAWS.h
io/s3/aws/S3ContextAWS.cc
io/s3/aws/S3ContextAWS.h
io/s3/S3BucketName.cc
io/s3/S3BucketName.h
io/s3/S3Client.cc
io/s3/S3Client.h
io/s3/S3Config.cc
io/s3/S3Config.h
io/s3/S3Credential.cc
io/s3/S3Credential.h
io/s3/S3Exception.cc
io/s3/S3Exception.h
io/s3/S3Handle.cc
io/s3/S3Handle.h
io/s3/S3Name.cc
io/s3/S3Name.h
io/s3/S3ObjectName.cc
io/s3/S3ObjectName.h
io/s3/S3Session.cc
io/s3/S3Session.h
io/s3/S3URIManager.cc
io/s3/S3URIManager.h
)
endif( eckit_HAVE_AWS_S3 )

list( APPEND eckit_filesystem_srcs
filesystem/BasePathName.cc
Expand Down Expand Up @@ -960,9 +961,9 @@ ecbuild_add_library(
"${BZIP2_INCLUDE_DIRS}"
"${AEC_INCLUDE_DIRS}"
"${RADOS_INCLUDE_DIRS}"
"${AWSSDK_INCLUDE_DIRS}"
"${OPENSSL_INCLUDE_DIR}"
"${AIO_INCLUDE_DIRS}"
"${AWSSDK_INCLUDE_DIRS}"

PRIVATE_LIBS
"${SNAPPY_LIBRARIES}"
Expand Down
37 changes: 20 additions & 17 deletions src/eckit/io/s3/S3BucketName.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,27 @@
#include "eckit/filesystem/URI.h"
#include "eckit/io/s3/S3Client.h"
#include "eckit/io/s3/S3Exception.h"
#include "eckit/io/s3/S3Name.h"
#include "eckit/io/s3/S3ObjectName.h"
#include "eckit/log/Log.h"
#include "eckit/net/Endpoint.h"

#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

namespace eckit {

//----------------------------------------------------------------------------------------------------------------------

S3BucketName::S3BucketName(const URI& uri): S3Name(uri) {
S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) {
parse();
}

S3BucketName::S3BucketName(const net::Endpoint& endpoint, const std::string& bucket):
S3Name(endpoint, "/"), bucket_(bucket) { }
S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket)
: S3Name(endpoint, "/"), bucket_ {std::move(bucket)} { }

//----------------------------------------------------------------------------------------------------------------------

Expand All @@ -56,38 +65,32 @@ auto S3BucketName::makeObject(const std::string& object) const -> std::unique_pt
}

auto S3BucketName::exists() const -> bool {
return client()->bucketExists(bucket_);
return client().bucketExists(bucket_);
}

void S3BucketName::create() {
client()->createBucket(bucket_);
client().createBucket(bucket_);
}

void S3BucketName::destroy() {
client()->deleteBucket(bucket_);
client().deleteBucket(bucket_);
}

void S3BucketName::ensureCreated() {
try {
create();
}
catch (S3EntityAlreadyExists& e) {
LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl;
}
} catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; }
}

void S3BucketName::ensureDestroyed() {
try {
client()->emptyBucket(bucket_);
client()->deleteBucket(bucket_);
}
catch (S3EntityNotFound& e) {
LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl;
}
client().emptyBucket(bucket_);
client().deleteBucket(bucket_);
} catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; }
}

auto S3BucketName::listObjects() const -> std::vector<std::string> {
return client()->listObjects(bucket_);
return client().listObjects(bucket_);
}

//----------------------------------------------------------------------------------------------------------------------
Expand Down
12 changes: 9 additions & 3 deletions src/eckit/io/s3/S3BucketName.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,24 @@
#pragma once

#include "eckit/io/s3/S3Name.h"
#include "eckit/net/Endpoint.h"

#include <iosfwd>
#include <memory>
#include <string>
#include <vector>

namespace eckit {

class S3ObjectName;

//----------------------------------------------------------------------------------------------------------------------

class S3BucketName: public S3Name {
class S3BucketName : public S3Name {
public: // methods
explicit S3BucketName(const URI& uri);

S3BucketName(const net::Endpoint& endpoint, const std::string& bucket);
S3BucketName(const net::Endpoint& endpoint, std::string bucket);

auto makeObject(const std::string& object) const -> std::unique_ptr<S3ObjectName>;

Expand All @@ -46,7 +52,7 @@ class S3BucketName: public S3Name {

void ensureDestroyed();

/// @todo: return S3 object iterator but first add prefix
/// @todo return S3 object iterator but first add prefix
auto listObjects() const -> std::vector<std::string>;

auto asString() const -> std::string override;
Expand Down
33 changes: 16 additions & 17 deletions src/eckit/io/s3/S3Client.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,40 @@

#include "eckit/io/s3/S3Client.h"

#include "S3Client.h"
#include "eckit/io/s3/S3Exception.h"
#include "eckit/exception/Exceptions.h"
#include "eckit/io/s3/S3Config.h"
#include "eckit/io/s3/S3Session.h"
#include "eckit/io/s3/aws/S3ClientAWS.h"
#include "eckit/log/CodeLocation.h"

#include <memory>
#include <ostream>
#include <utility>

namespace eckit {

//----------------------------------------------------------------------------------------------------------------------

S3Client::S3Client(const S3Config& config): config_(config) { }

S3Client::~S3Client() = default;
S3Client::S3Client(S3Config config) : config_ {std::move(config)} { }

//----------------------------------------------------------------------------------------------------------------------

void S3Client::print(std::ostream& out) const {
out << "S3Client[config=" << config_ << "]";
}
auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr<S3Client> {

//----------------------------------------------------------------------------------------------------------------------
if (config.backend == S3Backend::AWS) { return std::make_unique<S3ClientAWS>(config); }

auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr<S3Client> {
if (config.type == S3Types::AWS) { return std::make_shared<S3ClientAWS>(config); }
throw S3SeriousBug("Unkown S3 client type!", Here());
throw UserError("Unsupported S3 backend! Supported backend = AWS ", Here());
}

//----------------------------------------------------------------------------------------------------------------------

auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr<S3Client> {
if (config.type == S3Types::AWS) { return std::make_unique<S3ClientAWS>(config); }
throw S3SeriousBug("Unkown S3 client type!", Here());
void S3Client::print(std::ostream& out) const {
out << "S3Client[config=" << config_ << "]";
}

auto S3Client::endpoint() const -> const net::Endpoint& {
return config_.endpoint;
std::ostream& operator<<(std::ostream& out, const S3Client& client) {
client.print(out);
return out;
}

//----------------------------------------------------------------------------------------------------------------------
Expand Down
42 changes: 26 additions & 16 deletions src/eckit/io/s3/S3Client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
#pragma once

#include "eckit/io/s3/S3Config.h"
#include "eckit/memory/NonCopyable.h"

#include <cstdint>
#include <iosfwd>
#include <memory>
#include <string>
#include <vector>

namespace eckit {
Expand All @@ -31,15 +33,19 @@ class S3Context;

//----------------------------------------------------------------------------------------------------------------------

class S3Client: private NonCopyable {
class S3Client {
public: // methods
virtual ~S3Client();

static auto makeShared(const S3Config& config) -> std::shared_ptr<S3Client>;
S3Client(const S3Client&) = delete;
S3Client& operator=(const S3Client&) = delete;
S3Client(S3Client&&) = default;
S3Client& operator=(S3Client&&) = default;
virtual ~S3Client() = default;

static auto makeUnique(const S3Config& config) -> std::unique_ptr<S3Client>;

virtual auto endpoint() const -> const net::Endpoint&;
static auto makeShared(const S3Config& config) -> std::shared_ptr<S3Client> { return makeUnique(config); }

auto config() const -> const S3Config& { return config_; }

virtual void createBucket(const std::string& bucket) const = 0;

Expand All @@ -51,11 +57,16 @@ class S3Client: private NonCopyable {

virtual auto listBuckets() const -> std::vector<std::string> = 0;

virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer,
uint64_t length) const -> long long = 0;
virtual auto putObject(const std::string& bucket,
const std::string& object,
const void* buffer,
uint64_t length) const -> long long = 0;

virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset,
uint64_t length) const -> long long = 0;
virtual auto getObject(const std::string& bucket,
const std::string& object,
void* buffer,
uint64_t offset,
uint64_t length) const -> long long = 0;

virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0;

Expand All @@ -67,16 +78,15 @@ class S3Client: private NonCopyable {

virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0;

friend std::ostream& operator<<(std::ostream& out, const S3Client& client) {
client.print(out);
return out;
}

protected: // methods
explicit S3Client(const S3Config& config);
S3Client();

explicit S3Client(S3Config config);

virtual void print(std::ostream& out) const;

friend std::ostream& operator<<(std::ostream& out, const S3Client& client);

private: // members
S3Config config_;
};
Expand Down
Loading

0 comments on commit 536785a

Please sign in to comment.