Skip to content

Commit

Permalink
eckit::codec
Browse files Browse the repository at this point in the history
  • Loading branch information
pmaciel committed Nov 1, 2023
1 parent 8e6de4e commit 02d5729
Show file tree
Hide file tree
Showing 41 changed files with 303 additions and 309 deletions.
13 changes: 7 additions & 6 deletions src/eckit/codec/Data.cc
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ Data::Data(void* p, size_t size) :
buffer_(p, size), size_(size) {}

std::uint64_t Data::write(Stream& out) const {
if (size()) {
if (size() > 0) {
ASSERT(buffer_.size() >= size());
return out.write(buffer_.data(), size());
}
Expand All @@ -44,24 +44,25 @@ std::uint64_t Data::read(Stream& in, size_t size) {


void Data::compress(const std::string& compression) {
if (size_) {
if (size_ > 0) {
auto compressor = std::unique_ptr<Compressor>(CompressorFactory::instance().build(compression));
if (dynamic_cast<NoCompressor*>(compressor.get())) {
if (dynamic_cast<NoCompressor*>(compressor.get()) != nullptr) {
return;
}
Buffer compressed(size_t(1.2 * size_));

Buffer compressed(static_cast<size_t>(1.2 * static_cast<double>(size_)));
size_ = compressor->compress(buffer_, size_, compressed);
buffer_ = std::move(compressed);
}
}

void Data::decompress(const std::string& compression, size_t uncompressed_size) {
auto compressor = std::unique_ptr<Compressor>(CompressorFactory::instance().build(compression));
if (dynamic_cast<NoCompressor*>(compressor.get())) {
if (dynamic_cast<NoCompressor*>(compressor.get()) != nullptr) {
return;
}

Buffer uncompressed(size_t(1.2 * uncompressed_size));
Buffer uncompressed(static_cast<size_t>(1.2 * static_cast<double>(uncompressed_size)));
compressor->uncompress(buffer_, size_, uncompressed, uncompressed_size);
size_ = uncompressed_size;
buffer_ = std::move(uncompressed);
Expand Down
27 changes: 16 additions & 11 deletions src/eckit/codec/FileStream.cc
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,14 @@ class FileHandle : public eckit::FileHandle {
FileHandle(const PathName& path, const std::string& openmode) :
FileHandle(path, openmode[0]) {}

FileHandle(const FileHandle&) = delete;
FileHandle(FileHandle&&) = delete;

~FileHandle() override { close(); }

void operator=(const FileHandle&) = delete;
void operator=(FileHandle&&) = delete;

private:
bool closed_{false};
};
Expand All @@ -79,11 +85,19 @@ class FileHandle : public eckit::FileHandle {
/// - Automatic opening and closing of file
class PooledHandle : public eckit::PooledHandle {
public:
PooledHandle(const PathName& path) :
explicit PooledHandle(const PathName& path) :
eckit::PooledHandle(path), path_(path) {
openForRead();
}

PooledHandle(const PooledHandle&) = delete;
PooledHandle(PooledHandle&&) = delete;

~PooledHandle() override { close(); }

void operator=(const PooledHandle&) = delete;
void operator=(PooledHandle&&) = delete;

PathName path_;
};

Expand All @@ -92,16 +106,7 @@ class PooledHandle : public eckit::PooledHandle {
//---------------------------------------------------------------------------------------------------------------------

FileStream::FileStream(const PathName& path, char openmode) :
Stream([&path, &openmode]() -> DataHandle* {
DataHandle* datahandle;
if (openmode == 'r') {
datahandle = new PooledHandle(path);
}
else {
datahandle = new FileHandle(path, openmode);
}
return datahandle;
}()) {
Stream(openmode == 'r' ? static_cast<DataHandle*>(new PooledHandle(path)) : new FileHandle(path, openmode)) {
if (openmode == 'r') {
// Keep the PooledHandle alive until the end of active session
Session::store(*this);
Expand Down
14 changes: 7 additions & 7 deletions src/eckit/codec/FileStream.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,25 +36,25 @@ enum class Mode

class FileStream : public Stream {
public:
FileStream(const PathName& path, Mode openmode);
FileStream(const PathName& path, char openmode);
FileStream(const PathName& path, const std::string& openmode);
FileStream(const PathName&, Mode openmode);
FileStream(const PathName&, char openmode);
FileStream(const PathName&, const std::string& openmode);
};

//---------------------------------------------------------------------------------------------------------------------

class InputFileStream : public FileStream {
public:
InputFileStream(const PathName& path);
explicit InputFileStream(const PathName&);
};

//---------------------------------------------------------------------------------------------------------------------

class OutputFileStream : public FileStream {
public:
OutputFileStream(const PathName& path, Mode openmode = Mode::write);
OutputFileStream(const PathName& path, const std::string& openmode);
OutputFileStream(const PathName& path, char openmode);
explicit OutputFileStream(const PathName&, Mode openmode = Mode::write);
OutputFileStream(const PathName&, const std::string& openmode);
OutputFileStream(const PathName&, char openmode);
void close();
};

Expand Down
12 changes: 6 additions & 6 deletions src/eckit/codec/Metadata.cc
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ size_t uncompressed_size(const Metadata& m) {
if (m.has("data.size")) {
return m.getUnsigned("data.size");
}
else if (m.has("type")) {
if (m.getString("type") == "array") {
ArrayMetadata array(m);
return array.bytes();
}

if (m.has("type") && m.getString("type") == "array") {
ArrayMetadata array(m);
return array.bytes();
}

std::stringstream err;
err << "Could not compute uncompressed data size from metadata \n";
write(m, err);
Expand All @@ -58,7 +58,7 @@ void write(const Metadata& metadata, Stream& out) {

void Metadata::link(Metadata&& linked) {
std::string initial_link = link();
ASSERT(initial_link.size());
ASSERT(!initial_link.empty());

data = std::move(linked.data);
record = std::move(linked.record);
Expand Down
9 changes: 4 additions & 5 deletions src/eckit/codec/Metadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ class Metadata : public LocalConfiguration {
public:
using LocalConfiguration::LocalConfiguration;

Metadata() :
LocalConfiguration() {}
Metadata() = default;

Link link() const { return Link{getString("link", "")}; }

Expand All @@ -59,8 +58,8 @@ class Metadata : public LocalConfiguration {
// extended LocalConfiguration:
using LocalConfiguration::set;
Metadata& set(const LocalConfiguration& other) {
Value& root = const_cast<Value&>(get());
auto& other_root = other.get();
auto& root = const_cast<Value&>(get());
const auto& other_root = other.get();
std::vector<std::string> other_keys;
fromValue(other_keys, other_root.keys());
for (auto& key : other_keys) {
Expand All @@ -81,7 +80,7 @@ class Metadata : public LocalConfiguration {


Metadata& remove(const std::string& name) {
Value& root = const_cast<Value&>(get());
auto& root = const_cast<Value&>(get());
root.remove(name);
return *this;
}
Expand Down
2 changes: 1 addition & 1 deletion src/eckit/codec/ReadRequest.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ static std::string stream_path(Stream stream) {
ReadRequest::ReadRequest(const std::string& URI, Decoder* decoder) :
uri_(URI), decoder_(decoder), item_(new RecordItem()) {
do_checksum_ = defaults::checksum_read();
ASSERT(uri_.size());
ASSERT(!uri_.empty());
}

ReadRequest::ReadRequest(Stream stream, size_t offset, const std::string& key, Decoder* decoder) :
Expand Down
6 changes: 3 additions & 3 deletions src/eckit/codec/ReadRequest.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ class ReadRequest {
ReadRequest(const RecordItem::URI& URI, T& value) :
ReadRequest{URI.str(), value} {}

ReadRequest() = delete;
ReadRequest(const ReadRequest&) = delete;

~ReadRequest();

void read();
Expand All @@ -57,9 +60,6 @@ class ReadRequest {
ReadRequest(const std::string& URI, Decoder* decoder);
ReadRequest(Stream, size_t offset, const std::string& key, Decoder*);

ReadRequest() = delete;
ReadRequest(const ReadRequest&) = delete;

Stream stream_;
size_t offset_;
std::string key_;
Expand Down
26 changes: 11 additions & 15 deletions src/eckit/codec/Record.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace {

template <typename IStream, typename Struct>
inline size_t read_struct(IStream& in, Struct& s) {
static_assert(Struct::bytes == sizeof(Struct), "");
static_assert(Struct::bytes == sizeof(Struct));
return in.read(reinterpret_cast<char*>(&s), sizeof(Struct));
}

Expand All @@ -48,13 +48,9 @@ inline Struct read_struct(IStream& in) {
//---------------------------------------------------------------------------------------------------------------------

Endian RecordHead::endian() const {
if (magic_number == 1234) {
return Endian::native;
}
else if (magic_number == 3523477504) {
return Endian::swapped;
}
throw Exception("Mixed endianness is not supported", Here());
return magic_number == 1234 ? Endian::native
: magic_number == 3523477504 ? Endian::swapped
: throw Exception("Mixed endianness is not supported", Here());
}

//---------------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -196,7 +192,7 @@ Record& Record::read(Stream& in, bool read_to_end) {
"]");
}
std::string metadata_str;
metadata_str.resize(size_t(r.metadata_length) - sizeof(RecordMetadataSection::Begin) -
metadata_str.resize(static_cast<size_t>(r.metadata_length) - sizeof(RecordMetadataSection::Begin) -
sizeof(RecordMetadataSection::End));
if (in.read(const_cast<char*>(metadata_str.data()), metadata_str.size()) != metadata_str.size()) {
throw InvalidRecord("Unexpected EOF reached");
Expand Down Expand Up @@ -232,10 +228,10 @@ Record& Record::read(Stream& in, bool read_to_end) {
throw InvalidRecord("Data index section is not valid. Invalid section begin marker: [" + index_begin.str() +
"]");
}
const auto index_length =
(size_t(r.index_length) - sizeof(RecordDataIndexSection::Begin) - sizeof(RecordDataIndexSection::End));
const auto index_size = index_length / sizeof(RecordDataIndexSection::Entry);
auto& data_sections = record_->data_sections;
const auto index_length = (static_cast<size_t>(r.index_length) - sizeof(RecordDataIndexSection::Begin) -
sizeof(RecordDataIndexSection::End));
const auto index_size = index_length / sizeof(RecordDataIndexSection::Entry);
auto& data_sections = record_->data_sections;
data_sections.resize(index_size);
if (in.read(data_sections.data(), index_length) != index_length) {
throw InvalidRecord("Unexpected EOF reached");
Expand Down Expand Up @@ -278,8 +274,8 @@ void ParsedRecord::parse() {
item.data.section(item.getInt("data.section", 0));
item.data.endian(head.endian());
item.data.compression(item.getString("data.compression.type", "none"));
if (item.data.section()) {
auto& data_section = data_sections.at(size_t(item.data.section() - 1));
if (item.data.section() != 0) {
auto& data_section = data_sections.at(static_cast<size_t>(item.data.section() - 1));
item.data.checksum(data_section.checksum);
item.data.compressed_size(data_section.length - sizeof(RecordDataSection::Begin) -
sizeof(RecordDataSection::End));
Expand Down
13 changes: 7 additions & 6 deletions src/eckit/codec/Record.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,12 @@ class Record {
std::string str() const;
std::string path;
std::uint64_t offset;
URI() = default;
URI(const std::string& _path, std::uint64_t _offset = 0) :

URI() = default;
URI(const URI&) = default;

explicit URI(const std::string& _path, std::uint64_t _offset = 0) :
path(_path), offset(_offset) {}
URI(const URI& other) :
path(other.path), offset(other.offset) {}
};

private:
Expand All @@ -52,7 +53,7 @@ class Record {

bool empty() const;

Record& read(Stream& in, bool verify_end = false);
Record& read(Stream&, bool read_to_end = false);

const Metadata& metadata(const std::string& key) const;

Expand All @@ -68,7 +69,7 @@ class Record {

bool has(const std::string& key);

operator const ParsedRecord&() const;
explicit operator const ParsedRecord&() const;
};

//---------------------------------------------------------------------------------------------------------------------
Expand Down
4 changes: 2 additions & 2 deletions src/eckit/codec/RecordItem.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ namespace eckit::codec {
struct RecordItem {
public:
struct URI {
URI(const std::string& uri);
explicit URI(const std::string& uri);
URI(const std::string& _path, std::uint64_t _offset, const std::string& _key);
std::string str() const;
operator std::string() { return str(); }
operator std::string() const { return str(); }
std::string path;
std::uint64_t offset;
std::string key;
Expand Down
Loading

0 comments on commit 02d5729

Please sign in to comment.