diff --git a/VERSION b/VERSION index c514bd85..bc80560f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.6 +1.5.0 diff --git a/src/odc/ODBAPISettings.cc b/src/odc/ODBAPISettings.cc index cb49a208..bf513c0b 100644 --- a/src/odc/ODBAPISettings.cc +++ b/src/odc/ODBAPISettings.cc @@ -18,6 +18,7 @@ #include "eckit/log/Log.h" #include "eckit/thread/ThreadSingleton.h" #include "eckit/utils/StringTools.h" +#include "eckit/sql/LibEcKitSQL.h" #include "odc/LibOdc.h" #include "odc/ODBAPISettings.h" @@ -88,12 +89,21 @@ string odc::ODBAPISettings::fileInHome(const string& fileName) void odc::ODBAPISettings::treatIntegersAsDoubles(bool flag) { integersAsDoubles_ = flag; + LibEcKitSQL::instance().treatIntegersAsDoubles(flag); } bool odc::ODBAPISettings::integersAsDoubles() const { return integersAsDoubles_; } +bool odc::ODBAPISettings::fullyQualifySQLColumnNames() const { + return LibEcKitSQL::instance().fullyQualifyColumnNames(); +} + +void odc::ODBAPISettings::fullyQualifySQLColumnNames(bool flag) { + LibEcKitSQL::instance().fullyQualifyColumnNames(flag); +} + void debugMeNow() { Log::info() << "Debug me now" << endl; odc::ODBAPISettings::debug = true; diff --git a/src/odc/ODBAPISettings.h b/src/odc/ODBAPISettings.h index 64c5aa37..95d91300 100644 --- a/src/odc/ODBAPISettings.h +++ b/src/odc/ODBAPISettings.h @@ -41,6 +41,9 @@ class ODBAPISettings : private eckit::NonCopyable { void treatIntegersAsDoubles(bool flag); bool integersAsDoubles() const; + bool fullyQualifySQLColumnNames() const; + void fullyQualifySQLColumnNames(bool flag); + static bool debug; private: diff --git a/src/odc/WriterBufferingIterator.cc b/src/odc/WriterBufferingIterator.cc index 95feae79..1e79d52a 100644 --- a/src/odc/WriterBufferingIterator.cc +++ b/src/odc/WriterBufferingIterator.cc @@ -40,6 +40,7 @@ WriterBufferingIterator::WriterBufferingIterator(Owner &owner, DataHandle *dh, b columnOffsets_(0), columnByteSizes_(0), nrows_(0), + nrowsInFrame_(0), path_(owner.path()), initialisedColumns_(false), properties_(), @@ -63,6 +64,7 @@ WriterBufferingIterator::WriterBufferingIterator(Owner &owner, DataHandle &dh, b columnOffsets_(0), columnByteSizes_(0), nrows_(0), + nrowsInFrame_(0), path_(owner.path()), initialisedColumns_(false), properties_(), @@ -155,6 +157,7 @@ void WriterBufferingIterator::allocRowsBuffer() rowByteSize_ = rowDataSizeDoubles() * sizeof(double); rowsBuffer_ = Buffer(rowsBufferSize_ * rowByteSize_); nextRowInBuffer_ = reinterpret_cast(rowsBuffer_.data()); + nrowsInFrame_ = 0; } void WriterBufferingIterator::writeHeader() @@ -359,6 +362,7 @@ void WriterBufferingIterator::flush() // Reset the write buffers nextRowInBuffer_ = reinterpret_cast(rowsBuffer_.data()); + nrowsInFrame_ = 0; // This is a bad place to be. We need to reset the coders in the columns, not clone // the existing ones (which have been optimised). diff --git a/src/odc/WriterBufferingIterator.h b/src/odc/WriterBufferingIterator.h index 32ada60d..97d857fb 100644 --- a/src/odc/WriterBufferingIterator.h +++ b/src/odc/WriterBufferingIterator.h @@ -98,6 +98,8 @@ class WriterBufferingIterator : public eckit::HandleHolder std::vector outputFiles(); bool next(); + unsigned long long nrowsInFrame() const { return nrowsInFrame_; } + // If we are encoding strings, and the relevant string column size changes, we need // to restart the encoding process void flushAndResetColumnSizes(const std::map& resetColumnSizeDoubles); @@ -114,6 +116,7 @@ class WriterBufferingIterator : public eckit::HandleHolder size_t* columnOffsets_; // in doubles size_t* columnByteSizes_; unsigned long long nrows_; + unsigned long long nrowsInFrame_; eckit::PathName path_; @@ -172,8 +175,6 @@ void WriterBufferingIterator::pass1init(T& it, const T& end) template unsigned long WriterBufferingIterator::pass1(T& it, const T& end) { - LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1" << std::endl; - pass1init(it, end); writeHeader(); @@ -182,7 +183,6 @@ unsigned long WriterBufferingIterator::pass1(T& it, const T& end) { if (it->isNewDataset() && it->columns() != columns()) { - LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1: Change of input metadata." << std::endl; flush(); pass1init(it, end); writeHeader(); @@ -191,10 +191,8 @@ unsigned long WriterBufferingIterator::pass1(T& it, const T& end) writeRow(it->data(), it->columns().size()); } - LOG_DEBUG_LIB(LibOdc) << "Flushing rest of the buffer..." << std::endl; flush(); - LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1: processed " << nrows << " row(s)." << std::endl; ASSERT(close() == 0); return nrows; } diff --git a/src/odc/api/Odb.cc b/src/odc/api/Odb.cc index 3c618c26..e79da747 100644 --- a/src/odc/api/Odb.cc +++ b/src/odc/api/Odb.cc @@ -627,10 +627,22 @@ const std::map& Frame::properties() const { //---------------------------------------------------------------------------------------------------------------------- +bool Settings::treatIntegersAsDoubles() { + return odc::ODBAPISettings::instance().integersAsDoubles(); +} + void Settings::treatIntegersAsDoubles(bool flag) { odc::ODBAPISettings::instance().treatIntegersAsDoubles(flag); } +bool Settings::fullyQualifySQLColumnNames() { + return odc::ODBAPISettings::instance().fullyQualifySQLColumnNames(); +} + +void Settings::fullyQualifySQLColumnNames(bool flag) { + odc::ODBAPISettings::instance().fullyQualifySQLColumnNames(flag); +} + void Settings::setIntegerMissingValue(long val) { odc::MDI::integerMDI(val); } diff --git a/src/odc/api/Odb.h b/src/odc/api/Odb.h index 02c1ae6d..6e0c50dc 100644 --- a/src/odc/api/Odb.h +++ b/src/odc/api/Odb.h @@ -50,11 +50,23 @@ namespace api { class Settings { public: // methods + /** Determine whether integers are treated as doubles across the interface and internally + */ + static bool treatIntegersAsDoubles(); + /** Sets treatment of integers in ODB-2 data as doubles * \param flag Whether to treat integers as doubles (*true*) or longs (*false*) */ static void treatIntegersAsDoubles(bool flag); + /** Determine if column names is SQL query results are fully qualified, or returned as queried + */ + static bool fullyQualifySQLColumnNames(); + + /** Set if column names is SQL query results are fully qualified, or returned as queried + */ + static void fullyQualifySQLColumnNames(bool flag); + /** Returns the value that identifies a missing integer * \returns Missing integer value */ diff --git a/src/odc/api/odc.cc b/src/odc/api/odc.cc index f12c50e2..65b2b1da 100644 --- a/src/odc/api/odc.cc +++ b/src/odc/api/odc.cc @@ -228,6 +228,15 @@ int odc_integer_behaviour(int integerBehaviour) { }); } +int odc_sql_column_behaviour(int columnBehaviour) { + return wrapApiFunction([columnBehaviour] { + if (columnBehaviour != ODC_SQL_COLUMNS_FULLY_QUALIFIED && columnBehaviour != ODC_SQL_COLUMNS_AS_REFERENCED) { + throw SeriousBug("ODC column behaviour must be either ODC_SQL_COLUMNS_FULLY_QUALIFIED or ODC_SQL_COLUMNS_AS_REFERENCED", Here()); + } + Settings::fullyQualifySQLColumnNames(columnBehaviour == ODC_SQL_COLUMNS_FULLY_QUALIFIED); + }); +} + int odc_set_failure_handler(odc_failure_handler_t handler, void* context) { return wrapApiFunction([handler, context] { g_failure_handler = handler; diff --git a/src/odc/api/odc.h b/src/odc/api/odc.h index 1c48fdd6..24322c3e 100644 --- a/src/odc/api/odc.h +++ b/src/odc/api/odc.h @@ -33,22 +33,35 @@ extern "C" { /** @{ */ /** Represent integers as doubles in the API (default) */ -const int ODC_INTEGERS_AS_DOUBLES = 1; // this is the default +const int ODC_INTEGERS_AS_DOUBLES = 1; /* this is the default */ /** Represent integers as 64-bit integers in the API */ const int ODC_INTEGERS_AS_LONGS = 2; +/** Always fully qualify column names, including table names, in SQL query results */ +const int ODC_SQL_COLUMNS_FULLY_QUALIFIED = 1; /* this is the default */ + +/** Always fully qualify column names, including table names, in SQL query results */ +const int ODC_SQL_COLUMNS_AS_REFERENCED = 2; + /** Initialises API, must be called before any other function * \note This is only required if being used from a context where **eckit::Main()** is not otherwise initialised. * \returns Return code (#OdcErrorValues) */ int odc_initialise_api(); + /** Sets treatment of integers in the odc API * \param integerBehaviour Desired integer behaviour (#ODC_INTEGERS_AS_DOUBLES #ODC_INTEGERS_AS_LONGS) * \returns Return code (#OdcErrorValues) */ int odc_integer_behaviour(int integerBehaviour); +/** Sets behaviour of column naming in SQL queries + * \param columnBehaviour Desired column naming behaviour (#ODC_SQL_COLUMNS_FULLY_QUALIFIED #ODC_SQL_COLUMNS_AS_REFERENCED) + * \returns Return code (#OdcErrorValues) + */ +int odc_sql_column_behaviour(int columnBehaviour); + /** @} */ @@ -90,8 +103,6 @@ enum OdcErrorValues { */ const char* odc_error_string(int err); -// int odc_abort_on_failure(bool abort); ///< @todo to remove - /** Error handler callback function signature * \param context Error handler context * \param error_code Error code (#OdcErrorValues) @@ -139,8 +150,8 @@ int odc_column_type_count(int* count); */ int odc_column_type_name(int type, const char** type_name); -/// @todo In the top CMakelists.txt assert that in this system C long is 64 bit -/// @todo In the top CMakelists.txt assert that in this system C double is 64 bit +/** @todo In the top CMakelists.txt assert that in this system C long is 64 bit */ +/** @todo In the top CMakelists.txt assert that in this system C double is 64 bit */ /** Sets the value that identifies a missing integer in the API @@ -529,7 +540,7 @@ int odc_encoder_set_rows_per_frame(odc_encoder_t* encoder, long rows_per_frame); */ int odc_encoder_set_data_array(odc_encoder_t* encoder, const void* data, long width, long height, int columnMajorWidth); -/// @todo implement the int columnMajorWidth in the above function +/** @todo implement the int columnMajorWidth in the above function */ /** Adds a data column to current encoder * \param encoder Encoder instance diff --git a/src/odc/core/ThreadSharedDataHandle.cc b/src/odc/core/ThreadSharedDataHandle.cc index d6eff5d0..a7587e83 100644 --- a/src/odc/core/ThreadSharedDataHandle.cc +++ b/src/odc/core/ThreadSharedDataHandle.cc @@ -113,6 +113,11 @@ void ThreadSharedDataHandle::close() { internal_->dh_->close(); } +eckit::Length ThreadSharedDataHandle::size() { + ASSERT(internal_); + return internal_->dh_->size(); +} + eckit::Length ThreadSharedDataHandle::estimate() { ASSERT(internal_); return internal_->dh_->estimate(); diff --git a/src/odc/core/ThreadSharedDataHandle.h b/src/odc/core/ThreadSharedDataHandle.h index 3c0ce418..150563a5 100644 --- a/src/odc/core/ThreadSharedDataHandle.h +++ b/src/odc/core/ThreadSharedDataHandle.h @@ -54,6 +54,7 @@ class ThreadSharedDataHandle : public eckit::DataHandle { void close() override; eckit::Length estimate() override; + eckit::Length size() override; eckit::Offset position() override; eckit::Offset seek(const eckit::Offset&) override; diff --git a/src/odc/tools/ODAHeaderTool.cc b/src/odc/tools/ODAHeaderTool.cc index 33ac8d11..0354a5dc 100644 --- a/src/odc/tools/ODAHeaderTool.cc +++ b/src/odc/tools/ODAHeaderTool.cc @@ -52,8 +52,6 @@ class OffsetsPrinter : public MDPrinter { Length length (tbl.nextPosition() - tbl.startPosition()); o << offset << " " << length << " " << tbl.rowCount() << " " << tbl.columnCount() << std::endl; } -private: - unsigned long headerCount_; }; class DDLPrinter : public MDPrinter { diff --git a/tests/api/odc_encode_custom.cc b/tests/api/odc_encode_custom.cc index e999a5dc..59dbfb96 100644 --- a/tests/api/odc_encode_custom.cc +++ b/tests/api/odc_encode_custom.cc @@ -139,13 +139,13 @@ int main(int argc, char** argv) { // Define all column names, their types and sizes std::vector columns = { - ColumnInfo{std::string("expver"), ColumnType(STRING), 8}, - ColumnInfo{std::string("date@hdr"), ColumnType(INTEGER), sizeof(int64_t)}, - ColumnInfo{std::string("statid@hdr"), ColumnType(STRING), 8}, - ColumnInfo{std::string("wigos@hdr"), ColumnType(STRING), 16}, - ColumnInfo{std::string("obsvalue@body"), ColumnType(REAL), sizeof(double)}, - ColumnInfo{std::string("integer_missing"), ColumnType(INTEGER), sizeof(int64_t)}, - ColumnInfo{std::string("double_missing"), ColumnType(REAL), sizeof(double)}, + ColumnInfo{std::string("expver"), ColumnType(STRING), 8, {}}, + ColumnInfo{std::string("date@hdr"), ColumnType(INTEGER), sizeof(int64_t), {}}, + ColumnInfo{std::string("statid@hdr"), ColumnType(STRING), 8, {}}, + ColumnInfo{std::string("wigos@hdr"), ColumnType(STRING), 16, {}}, + ColumnInfo{std::string("obsvalue@body"), ColumnType(REAL), sizeof(double), {}}, + ColumnInfo{std::string("integer_missing"), ColumnType(INTEGER), sizeof(int64_t), {}}, + ColumnInfo{std::string("double_missing"), ColumnType(REAL), sizeof(double), {}}, ColumnInfo{std::string("bitfield_column"), ColumnType(BITFIELD), sizeof(int64_t), bitfields}, }; diff --git a/tests/api/read.cc b/tests/api/read.cc index bc45a7bf..1d441fe1 100644 --- a/tests/api/read.cc +++ b/tests/api/read.cc @@ -48,9 +48,9 @@ void test_generate_odb_properties(const std::string &path, int propertiesMode = // Define all column names, their types and sizes std::vector columns = { - {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8}, - {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t)}, - {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double)}, + {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8, {}}, + {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t), {}}, + {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, }; // Set a custom data layout and data array for each column @@ -154,10 +154,10 @@ void test_generate_odb_span(const std::string &path) { // Define all column names, their types and sizes std::vector columns = { - {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8}, - {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t)}, - {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double)}, - {std::string("missing_value"), odc::api::ColumnType(odc::api::REAL), sizeof(double)}, + {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8, {}}, + {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t), {}}, + {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, + {std::string("missing_value"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, }; // Set a custom data layout and data array for each column and frame @@ -1140,7 +1140,7 @@ CASE("Where Span interface is used to read values without decoding") { } // Check string values - for (const std::string val : expver_vals) { + for (const std::string& val : expver_vals) { EXPECT(val == "xxxx"); } @@ -1164,6 +1164,8 @@ CASE("Where Span interface is used to read values without decoding") { CASE("Filter a subset of ODB-2 data") { + ASSERT(!odc::api::Settings::treatIntegersAsDoubles()); + eckit::FileHandle in("../2000010106-reduced.odb"); in.openForRead(); eckit::AutoClose close_in(in); @@ -1191,10 +1193,10 @@ CASE("Filter a subset of ODB-2 data") { size_t nframes = 0; size_t totalRows = 0; - odc::api::Frame frame = reader.next(); + odc::api::Frame test_frame = reader.next(); // Test second type of frame constructor via an existing copy - odc::api::Frame test_frame(frame); +// odc::api::Frame test_frame(frame); EXPECT(test_frame.rowCount() == 10); @@ -1228,9 +1230,43 @@ CASE("Filter a subset of ODB-2 data") { EXPECT(long(sub_frame.length()) == 487); // Test creation of a frame via the assignment operator - odc::api::Frame dummy_frame = frame; +// odc::api::Frame dummy_frame = frame; + +// EXPECT(dummy_frame.rowCount() == 10); +} + +// ------------------------------------------------------------------------------------------------------ + +CASE("Read from data with explicit SQL filter") { + + bool aggregated = true; + long rowlimit = -1; + const char* sql_filter ="select expver, date, lat, lon, obsvalue where rownumber() <= 10"; + odc::api::Reader reader("../2000010106-reduced.odb", aggregated, rowlimit, sql_filter); + + size_t nframes = 0; + size_t totalRows = 0; + odc::api::Frame frame = reader.next(); + + // Test second type of frame constructor via an existing copy + odc::api::Frame test_frame(frame); - EXPECT(dummy_frame.rowCount() == 10); + EXPECT(test_frame.rowCount() == 10); + + // Access encoded data directly from a frame + EXPECT(test_frame.encodedData().size() == 487); + + // Check if frame has expected columns. + EXPECT(test_frame.columnCount() == 5); + EXPECT(test_frame.hasColumn("expver")); + EXPECT(test_frame.hasColumn("date@hdr")); + EXPECT(test_frame.hasColumn("lat@hdr")); + EXPECT(test_frame.hasColumn("lon@hdr")); + EXPECT(test_frame.hasColumn("obsvalue@body")); + + // Check expected frame bounds + EXPECT(int(test_frame.offset()) == 0); + EXPECT(long(test_frame.length()) == 487); } // ------------------------------------------------------------------------------------------------------ diff --git a/tests/c_api/read.cc b/tests/c_api/read.cc index 8953ba77..a56b3482 100644 --- a/tests/c_api/read.cc +++ b/tests/c_api/read.cc @@ -66,9 +66,9 @@ void test_generate_odb(const std::string &path, int propertiesMode) { // Define all column names, their types and sizes std::vector columns = { - {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8}, - {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t)}, - {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double)}, + {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8, {}}, + {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t), {}}, + {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, }; // Set a custom data layout and data array for each column