Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support optimal encoding of constant string columns greater than 8 characters long #22

Merged
merged 1 commit into from
Aug 6, 2024
Merged

Support optimal encoding of constant string columns greater than 8 characters long #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/odc/codec/CodecOptimizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ int CodecOptimizer::setOptimalCodecs(core::MetaData& columns)
n = col.coder().numStrings();
if (n == 1 && col.coder().dataSizeDoubles() == 1)
codec = "constant_string";
else if (n == 1 && std::getenv("ODC_ENABLE_WRITING_LONG_STRING_CODEC") != NULL)
codec = "long_constant_string";
else if(n < 256)
codec = "int8_string";
else if(n < 65536)
Expand Down
1 change: 1 addition & 0 deletions src/odc/codec/Constant.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ namespace codec {
namespace {
core::IntegerCodecBuilder<CodecConstant> constantBuilder;
core::CodecBuilder<CodecConstantString> constantStringBuilder;
core::CodecBuilder<CodecLongConstantString> LongConstantStringBuilder;
}

//----------------------------------------------------------------------------------------------------------------------
Expand Down
53 changes: 53 additions & 0 deletions src/odc/codec/Constant.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#define odc_core_codec_Constant_H

#include "odc/core/Codec.h"
#include "odc/codec/String.h"

namespace odc {
namespace codec {
Expand Down Expand Up @@ -74,6 +75,25 @@ class CodecConstantString : public CodecConstant<ByteOrder, double> {
void save(core::DataStream<ByteOrder>& ds) override;
};

template<typename ByteOrder>
class CodecLongConstantString : public CodecChars<ByteOrder> {

public: // methods
constexpr static const char* codec_name() { return "long_constant_string"; }
CodecLongConstantString(api::ColumnType type) : CodecChars<ByteOrder>(type, codec_name()) {};
~CodecLongConstantString() override {}

private: // methods
unsigned char* encode(unsigned char* p, const double& d) override;
void decode(double* out) override;
void skip() override;

size_t numStrings() const override { return 1; }

void load(core::DataStream<ByteOrder>& ds) override;
void save(core::DataStream<ByteOrder>& ds) override;
};

//----------------------------------------------------------------------------------------------------------------------

// Implementation of Constant
Expand Down Expand Up @@ -151,6 +171,39 @@ void CodecConstantString<ByteOrder>::print(std::ostream& s) const {

//----------------------------------------------------------------------------------------------------------------------

// Implementation of LongConstantString

template <typename ByteOrder>
unsigned char* CodecLongConstantString<ByteOrder>::encode(unsigned char* p, const double&) {
return p;
}

template <typename ByteOrder>
void CodecLongConstantString<ByteOrder>::decode(double* out) {
::memset(out, 0, this->decodedSizeDoubles_*sizeof(double));
::memcpy(reinterpret_cast<char*>(out), &this->strings_[0][0], std::min(this->strings_[0].length(), this->decodedSizeDoubles_*sizeof(double)));
}

template <typename ByteOrder>
void CodecLongConstantString<ByteOrder>::skip() {}

template <typename ByteOrder>
void CodecLongConstantString<ByteOrder>::load(core::DataStream<ByteOrder>& ds) {
core::DataStreamCodec<ByteOrder>::load(ds);
std::string s;
ds.read(s);
this->decodedSizeDoubles_ = ((s.length()-1) / sizeof(double)) +1;
this->strings_.push_back(s);
}

template <typename ByteOrder>
void CodecLongConstantString<ByteOrder>::save(core::DataStream<ByteOrder>& ds) {
core::DataStreamCodec<ByteOrder>::save(ds);
ds.write(this->strings_[0]);
}

//----------------------------------------------------------------------------------------------------------------------

} // namespace codec
} // namespace odc

Expand Down
Loading