[WIP] created the new object hierarchy

src/ecom/common/om_general_utils_c.c

@@ -6,17 +6,53 @@
 #if __linux__
 #include <sys/sysinfo.h>
 
-void om_get_mem_usage(uint64_t* total_memory_of_system, uint64_t* system_usage, uint64_t* task_usage) {
+typedef struct KeyBits {
+
+   uint32_t sid: 1;
+   uint32_t paramId: 27;
+   uint32_t id: 26;
+   uint32_t levtype: 4;
+   uint32_t repres: 2;
+   uint32_t model: 2;
+   uint32_t precision: 2;
+
+} KeyBits_t;
+
+typedef union Key {
+    int64_t key;
+    KeyBits_t bits;
+} Key_t;
+
+void to_field_hash( int32_t paramId, int32_t id, int32_t levtype, int32_t repres, int32_t model, int32_t precision, int64_t* hash) {
+
+  Key_t k;
+
+  k.bits.sid = id >= 0 ? 1 : 0;
+  k.bits.paramId = (uint32_t)paramId;
+  k.bits.id = id >= 0 ? id : -id;
+  k.bits.levtype   = (uint32_t)levtype;
+  k.bits.repres    = (uint32_t)repres;
+  k.bits.model   = (uint32_t)model;
+  k.bits.precision = (uint32_t)precision;
+
+  *hash = k.key;
+
+ return;
+
+};
+
+
+void om_get_mem_usage( int64_t* total_memory_of_system, int64_t* system_usage, int64_t* task_usage) {
     // Retrieve total memory of the system
     struct sysinfo info;
     sysinfo(&info);
-    *total_memory_of_system = (uint64_t)info.totalram * info.mem_unit;
+    *total_memory_of_system = (int64_t)info.totalram * info.mem_unit;
 
     // Retrieve memory usage of the current process
     struct rusage usage;
     if (getrusage(RUSAGE_SELF, &usage) == 0) {
-        *system_usage = *total_memory_of_system - (uint64_t)info.freeram * info.mem_unit;
-        *task_usage = (uint64_t)usage.ru_maxrss * 1024;  // Convert to bytes
+        *system_usage = *total_memory_of_system - (int64_t)info.freeram * info.mem_unit;
+        *task_usage = (int64_t)usage.ru_maxrss * 1024;  // Convert to bytes
     }
     else {
         // Error handling

src/ecom/common/om_general_utils_mod.F90

@@ -37,9 +37,141 @@ MODULE OM_GENERAL_UTILS_MOD
 PUBLIC :: OM_READ_YAML_FROM_ENV
 PUBLIC :: OM_IS_LITTLE_ENDIAN
 PUBLIC :: OM_FINDLOC
+PUBLIC :: OM_FIELD_HASH
 
 CONTAINS
 
+!>
+!> @brief Compute a unique hash for a field.
+!>
+!> This function computes a unique hash for a field based on the field's ID, level type, representation.
+!>
+!> @attention the folder is "../" because by default each instance of the output manager run in
+!>            the folder calles io_serv.<procId>.d
+!>
+!> @param [out] OMYAML Name of the main YAML configuraiton file
+!>
+#define PP_PROCEDURE_TYPE 'FUNCTION'
+#define PP_PROCEDURE_NAME 'OM_FIELD_HASH'
+FUNCTION OM_FIELD_HASH( PARAM_ID, ID, LEVTYPE, REPRES, MODEL, PRECISION ) RESULT(HASH)
+
+  ! Symbols imported from intrinsic modules
+  USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_INT32_T
+  USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_INT64_T
+
+  ! Symbols imported from other modules within the project.
+  USE :: OM_CORE_MOD, ONLY: JPIB_K
+
+  ! Symbols imported by the preprocessor for debugging purposes
+  PP_DEBUG_USE_VARS
+
+  ! Symbols imported by the preprocessor for tracing purposes
+  PP_TRACE_USE_VARS
+
+IMPLICIT NONE
+
+  ! Dummy arguments
+  INTEGER(KIND=JPIB_K), INTENT(IN) :: PARAM_ID
+  INTEGER(KIND=JPIB_K), INTENT(IN) :: ID
+  INTEGER(KIND=JPIB_K), INTENT(IN) :: LEVTYPE
+  INTEGER(KIND=JPIB_K), INTENT(IN) :: REPRES
+  INTEGER(KIND=JPIB_K), INTENT(IN) :: MODEL
+  INTEGER(KIND=JPIB_K), INTENT(IN) :: PRECISION
+
+  ! Function Result
+  INTEGER(KIND=JPIB_K) :: HASH
+
+  ! Local variables
+  INTEGER(KIND=C_INT32_T) :: C_PARAM_ID
+  INTEGER(KIND=C_INT32_T) :: C_ID
+  INTEGER(KIND=C_INT32_T) :: C_LEVTYPE
+  INTEGER(KIND=C_INT32_T) :: C_REPRES
+  INTEGER(KIND=C_INT32_T) :: C_MODEL
+  INTEGER(KIND=C_INT32_T) :: C_PRECISION
+  INTEGER(KIND=C_INT64_T) :: C_HASH
+
+  ! Explicit interfaces
+  INTERFACE
+    SUBROUTINE C_TO_FIELD_HASH( C_PARAM_ID, C_ID, C_LEVTYPE, C_REPRES, C_MODEL, C_PRECISION, C_HASH ) BIND(C, NAME="to_field_hash")
+      USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_INT32_T
+      USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_INT64_T
+    IMPLICIT NONE
+      INTEGER(KIND=C_INT32_T), VALUE, INTENT(IN)  :: C_PARAM_ID
+      INTEGER(KIND=C_INT32_T), VALUE, INTENT(IN)  :: C_ID
+      INTEGER(KIND=C_INT32_T), VALUE, INTENT(IN)  :: C_LEVTYPE
+      INTEGER(KIND=C_INT32_T), VALUE, INTENT(IN)  :: C_REPRES
+      INTEGER(KIND=C_INT32_T), VALUE, INTENT(IN)  :: C_MODEL
+      INTEGER(KIND=C_INT32_T), VALUE, INTENT(IN)  :: C_PRECISION
+      INTEGER(KIND=C_INT64_T),        INTENT(OUT) :: C_HASH
+    END SUBROUTINE C_TO_FIELD_HASH
+  END INTERFACE
+
+  ! Local variables declared by the preprocessor for debugging purposes
+  PP_DEBUG_DECL_VARS
+
+  ! Local variables declared by the preprocessor for tracing purposes
+  PP_TRACE_DECL_VARS
+
+  ! Trace begin of procedure
+  PP_TRACE_ENTER_PROCEDURE()
+
+  C_PARAM_ID = INT(PARAM_ID,KIND=C_INT32_T)
+  C_ID = INT(ID,KIND=C_INT32_T)
+  C_LEVTYPE = INT(LEVTYPE,KIND=C_INT32_T)
+  C_REPRES = INT(REPRES,KIND=C_INT32_T)
+  C_MODEL = INT(MODEL,KIND=C_INT32_T)
+  C_PRECISION = INT(PRECISION,KIND=C_INT32_T)
+
+  ! Call the C function
+  CALL C_TO_FIELD_HASH( C_PARAM_ID, C_ID, C_LEVTYPE, C_REPRES, C_MODEL, C_PRECISION, C_HASH )
+
+  ! Cast to fortran datatype
+  HASH = INT( C_HASH, KIND=JPIB_K )
+
+  ! Trace end of procedure (on success)
+  PP_TRACE_EXIT_PROCEDURE_ON_SUCCESS()
+
+  ! Exit point
+  RETURN
+
+! Error handler
+PP_ERROR_HANDLER
+
+  ErrorHandler: BLOCK
+
+    ! Error handling variables
+    CHARACTER(LEN=:), ALLOCATABLE :: STR
+
+
+    ! HAndle different errors
+    SELECT CASE(ERRIDX)
+
+    CASE (1)
+      PP_DEBUG_CREATE_ERROR_MSG( STR, 'OUTPUT_MANAGER_YAML env. var. too long' )
+
+    CASE (2)
+      PP_DEBUG_CREATE_ERROR_MSG( STR, 'OUTPUT_MANAGER_YAML unable to find the file' )
+
+    CASE DEFAULT
+      PP_DEBUG_CREATE_ERROR_MSG( STR, 'Unhandled error' )
+
+    END SELECT
+
+    ! Trace end of procedure (on error)
+    PP_TRACE_EXIT_PROCEDURE_ON_ERROR()
+
+    ! Write the error message and stop the program
+    PP_DEBUG_ABORT( STR )
+
+  END BLOCK ErrorHandler
+
+  ! Exit point on error
+  RETURN
+
+END FUNCTION OM_FIELD_HASH
+#undef PP_PROCEDURE_NAME
+#undef PP_PROCEDURE_TYPE
+
 !>
 !> @brief Retrieves the output manager type from the 'OUTPUT_MANAGER_YAML' environment variable.
 !>

src/ecom/config_example/output-manager-config.yaml

@@ -43,14 +43,150 @@ encoding-rules:
 
   rules:
     - rule: 'grib1_surfaces_gridded'
-      filter:
+
+      filter: # mandatory
         levtype: ['sfc']
-        repres: ['gridded']
+        repres:  ['gridded']
         paramId: [ 31, 34, 49, 78, 134, 136, 137, 142, 144, 151, 165, 166, 168, 169, 175, 176, 177, 178, 179, 180, 181, 205, 228, 228029, 228216, 228218, 228219 ]
 
-      encode:
-        gribEdition: 1
-        packingType: 'grid_simple'
+      tag: 'GRIB1' # optional
+
+      mapping-rules: # optional
+        - recipe:
+          from:
+            paramId: 31
+            level: 0
+            levtype: 'sfc'
+          to:
+            paramId: 160350
+            level: 4
+            levtype: 'sfc'
+            values-scale-factor: 0.01
+
+      # grib-structure: # optional
+      #   local-definition-template-number: 2
+      #   grid-definition-template-number: 2
+      #   product-definition-template-number: 2
+      #   data-definition-template-number: 2
+      #
+      #
+      #
+      edition: # mandatory
+        use-paramId-ecmf: false
+        grib-edition: 1
+
+      packing: # optional
+        packing-type: 'grid_simple'
+
+
+encoding-rules:
+
+  default-rules:
+    -rule: 'default-rule-that-should-work-almost-always'
+       ...
+
+  special-rules:
+   - rule: 'grib1_surfaces_gridded'
+
+        # Select messages
+      - type: filter
+        match:
+          levtype: ['sfc']
+          repres:  ['gridded']
+          paramId: [ 31, 34, 49, 78, 134, 136, 137, 142, 144, 151, 165, 166, 168, 169, 175, 176, 177, 178, 179, 180, 181, 205, 228, 228029, 228216, 228218, 228219 ]
+
+        # This can be used to convert prefix to levtype
+      - type: mapping
+        recipes:
+          - recipe:
+            from:
+              paramId: 31
+              level: 0
+              levtype: 'sfc'
+            to:
+              paramId: 160350
+              level: 4
+              levtype: 'sfc'
+              values-scale-factor: 0.01
+
+        # Sample to be loaded fro this rule (should be shared between the rules)
+      - type: load-sample
+        sample-path: <>
+        sample-name: <>
+        options:
+          strip-values: true
+          strip-bitmask: true
+          strip-section2: true
+          strip-section3: true
+
+        # Everything that can be done once should be done once and then cached.
+        # First time a message with a specific hash arrive for the first time (not in the map),
+        # The sample is loaded and preset and then added to the map.
+      - type: lazy-preset
+        presets:
+
+          - type: local-definition-section
+            template-number: [2, <metkit::map-name> ]
+            options:
+              bla: <specific configurations to be set for the grib2 section2 number 2>
+
+          - type: grid-definition-section
+            template-number:  [40, <metkit::map-name> ]
+            options:
+              bla: ...
+
+          - type: product-definition-section
+            template-number: [ 1, <metkit::map-name> ]
+            options:
+              bla: ...
+
+          - type: data-definition-section
+            template-number: [ 2, <metkit::map-name> ]
+            options:
+              bla: ...
+
+        # When a fully preset template is in the map, the only euntime change are the values and the time configuration.
+        # Values may need a scaling due to the mapping (i.e. change units)
+      - type: runtime-configurations
+        runtime-cfg:
+          - type: time-configuration
+            options:
+              bla: ...
+
+          - type: values-configuration
+            options:
+              bla: ...
+
+
+      - type: sink
+        sinks:
+          - type: grib-msg-to-file
+            options:
+              bla: ...
+
+          - type: grib-msg-to-multio
+            option:
+              bla: ...
+
+          - type: grib-msg-to-multio
+            option:
+              bla: ...
+
+
+      # grib-structure: # optional
+      #   local-definition-template-number: 2
+      #   grid-definition-template-number: 2
+      #   product-definition-template-number: 2
+      #   data-definition-template-number: 2
+      #
+      #
+      #
+      edition: # mandatory
+        use-paramId-ecmf: false
+        grib-edition: 1
+
+      packing: # optional
+        packing-type: 'grid_simple'
 
 
     - rule: 'grib1_height_level_gridded'