diff --git a/diag_manager/diag_yaml_format.md b/diag_manager/diag_yaml_format.md index d9e93c359..d16628ddc 100644 --- a/diag_manager/diag_yaml_format.md +++ b/diag_manager/diag_yaml_format.md @@ -43,14 +43,14 @@ diag_files: ### 2.1 Global Section The diag_yaml requires “title” and the “baseDate”. - The **title** is a string that labels the diag yaml. The equivalent in the legacy diag_table would be the experiment. It is recommended that each diag_yaml have a separate title label that is descriptive of the experiment that is using it. -- The **basedate** is an array of 6 integers indicating the base_date in the format [year month day hour minute second]. +- The **base_date** is a comma-separated array of 6 integers indicating the base_date in the format [year, month, day, hour, minute, second]. Spacing is not significant. **Example:** In the YAML format: ```yaml title: ESM4_piControl -base_date: 2022 5 26 12 3 1 +base_date: [2022, 5, 26, 12, 3, 1] ``` In the legacy ascii format: @@ -281,7 +281,7 @@ The sub region can be listed under the sub_region section as a dashed array. The Bellow is a complete example of diag_table.yaml: ```yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: wild_card_name%4yr%2mo%2dy%2hr freq: 6 hours diff --git a/diag_manager/fms_diag_yaml.F90 b/diag_manager/fms_diag_yaml.F90 index a1c9b0b80..2edb5bb69 100644 --- a/diag_manager/fms_diag_yaml.F90 +++ b/diag_manager/fms_diag_yaml.F90 @@ -1667,13 +1667,14 @@ subroutine fms_diag_yaml_out() call fms_f2c_string( keys(1)%key2, 'base_date') basedate_loc = diag_yaml%get_basedate() tmpstr1 = ''; tmpstr2 = '' - tmpstr1 = string(basedate_loc(1)) + tmpstr1 = '[ '//string(basedate_loc(1)) tmpstr2 = trim(tmpstr1) do i=2, basedate_size tmpstr1 = string(basedate_loc(i)) - tmpstr2 = trim(tmpstr2) // ' ' // trim(tmpstr1) + tmpstr2 = trim(tmpstr2) // ', ' // trim(tmpstr1) enddo - call fms_f2c_string(vals(1)%val2, trim(tmpstr2)) + tmpstr1 = trim(tmpstr2) // ']' + call fms_f2c_string(vals(1)%val2, trim(tmpstr1)) call yaml_out_add_level2key('diag_files', keys(1)) key3_i = 0 !! tier 2 - diag files diff --git a/parser/yaml_output_functions.c b/parser/yaml_output_functions.c index e748a8272..b44f2ea35 100644 --- a/parser/yaml_output_functions.c +++ b/parser/yaml_output_functions.c @@ -89,7 +89,9 @@ void keyerror(yaml_event_t * event, yaml_emitter_t * emitter){ fprintf(stderr, "WARNING: YAML_OUTPUT: Failed to emit event %d: %s\n", event->type, emitter->problem); fprintf(stdout, "WARNING: YAML_OUTPUT: Failed to emit event %d: %s\n", event->type, emitter->problem); } -/* \breif Writes the key/value pairs of the fmsyamloutkeys and fmsyamloutvalues structs +/* \brief Writes the key/value pairs of the fmsyamloutkeys and fmsyamloutvalues structs + * \note If second value (val2) in struct starts with '[' it will be assumed the value is a yaml array + * There may be slight differences in spacing for array outputs vs what is read in. * \param emitter The libyaml emitter for this file * \param event The libyaml eent pointer * \param aindex The index of keys and vals that are being written currently @@ -124,11 +126,58 @@ void write_keys_vals_yaml (yaml_emitter_t * emitter, yaml_event_t * event , int keyerror(event, emitter); return; } - yaml_scalar_event_initialize(event, NULL, (yaml_char_t *)YAML_STR_TAG, - (yaml_char_t *)vals[aindex].val2, strlen(vals[aindex].val2), 1, 0, YAML_PLAIN_SCALAR_STYLE); - if (!yaml_emitter_emit(emitter, event)){ - keyerror(event, emitter); - return; + + // check if we're writing an array + if(vals[aindex].val2[0] != '['){ + yaml_scalar_event_initialize(event, NULL, NULL, + (yaml_char_t *)vals[aindex].val2, strlen(vals[aindex].val2), 1, 0, YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, event)){ + keyerror(event, emitter); + return; + } + } else { + // parse the string for individual elements + char *buff = vals[aindex].val2; + char cbuff[2]; // single char buffer for appending + cbuff[1] = '\0'; + char elements[16][255]; + int ecount=0; // count of elements + strcpy(elements[0], ""); + for(int i=1; i< strlen(buff)-1; i++){ + if(buff[i] != ' ' && buff[i] != ','){ + cbuff[0] = buff[i]; + strcat(elements[ecount], cbuff); + } + if(buff[i] == ','){ + ecount++; + if(ecount >= 16) { + printf("Error: element count in write_keys_vals_yaml greater than max value of 16"); + return; + } + strcpy(elements[ecount], ""); + } + } + // start flow sequence + yaml_sequence_start_event_initialize(event, NULL, NULL, 1, YAML_FLOW_SEQUENCE_STYLE); + if (!yaml_emitter_emit(emitter, event)){ + keyerror(event, emitter); + return; + } + // loop through and write elements + for (int i=0; i <= ecount; i++){ + yaml_scalar_event_initialize(event, NULL, NULL, + (yaml_char_t *)elements[i], strlen(elements[i]), 1, 0, YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, event)){ + keyerror(event, emitter); + return; + } + } + // end flow sequence + yaml_sequence_end_event_initialize(event); + if (!yaml_emitter_emit(emitter, event)){ + keyerror(event, emitter); + return; + } } } if (keys[aindex].key3[0] !='\0') { diff --git a/parser/yaml_parser.F90 b/parser/yaml_parser.F90 index 3a3c7f005..08e266598 100644 --- a/parser/yaml_parser.F90 +++ b/parser/yaml_parser.F90 @@ -70,6 +70,8 @@ module yaml_parser_mod interface !> @brief Private c function that opens and parses a yaml file (see yaml_parser_binding.c) +!! Any arrays (formatted as [a, b, c] or a multiline '-' tabbed list) will be +!! read in as a space-separated list of characters !! @return Flag indicating if the read was successful function open_and_parse_file_wrap(filename, file_id) bind(c) & result(error_code) diff --git a/parser/yaml_parser_binding.c b/parser/yaml_parser_binding.c index 778b6267d..a5ebe5c99 100644 --- a/parser/yaml_parser_binding.c +++ b/parser/yaml_parser_binding.c @@ -316,6 +316,9 @@ int open_and_parse_file_wrap(char *filename, int *file_id) char current_layername[255]; /* Name of the current block */ int i; /* To minimize the typing :) */ int j; /* To minimize the typing :) */ + bool is_flow_sequence = false;/* Flag indicating if within a flow sequence (aka a yaml array ie. [1, 2, 3, 4] or - tabbed list)*/ + bool is_first = false; /* Flag indicating if current item is the first item in the flow sequence*/ + char buff[255]; /* String buffer for individual array values*/ if (nfiles == 0 ) { @@ -356,6 +359,19 @@ int open_and_parse_file_wrap(char *filename, int *file_id) is_key = false; break; } + case YAML_FLOW_SEQUENCE_START_TOKEN: + { + is_key = false; + is_flow_sequence = true; + is_first = true; + break; + } + case YAML_FLOW_SEQUENCE_END_TOKEN: + { + is_key = false; + is_flow_sequence = false; + break; + } case YAML_BLOCK_ENTRY_TOKEN: { layer = layer + 1; @@ -386,19 +402,28 @@ int open_and_parse_file_wrap(char *filename, int *file_id) { if ( ! is_key) { - current_parent = parent[layer]; - strcpy(current_layername, ""); - key_count = key_count + 1; - i = key_count; - my_files.files[j].keys = realloc(my_files.files[j].keys, (i+1)*sizeof(key_value_pairs)); - my_files.files[j].keys[i].key_number=i; - my_files.files[j].keys[i].parent_key = current_parent; - strcpy(my_files.files[j].keys[i].parent_name, current_layername); - strcpy(my_files.files[j].keys[i].key, key_value); - strcpy(my_files.files[j].keys[i].value, token.data.scalar.value); - my_files.files[j].nkeys = key_count; - /* printf("----> LAYER:%i LAYER_NAME=%s PARENT:%i, KEYCOUNT:%i KEY: %s VALUE: %s \n", layer, current_layername, current_parent, key_count, key_value, token.data.scalar.value); */ - strcpy(key_value,""); + if (! is_flow_sequence || is_first) // if either a normal value or the first element in a flow sequence + { + current_parent = parent[layer]; + strcpy(current_layername, ""); + key_count = key_count + 1; + i = key_count; + my_files.files[j].keys = realloc(my_files.files[j].keys, (i+1)*sizeof(key_value_pairs)); + my_files.files[j].keys[i].key_number=i; + my_files.files[j].keys[i].parent_key = current_parent; + strcpy(my_files.files[j].keys[i].parent_name, current_layername); + strcpy(my_files.files[j].keys[i].key, key_value); + strcpy(my_files.files[j].keys[i].value, token.data.scalar.value); + my_files.files[j].nkeys = key_count; + /* printf("----> LAYER:%i LAYER_NAME=%s PARENT:%i, KEYCOUNT:%i KEY: %s VALUE: %s \n", layer, current_layername, current_parent, key_count, key_value, token.data.scalar.value); */ + strcpy(key_value,""); + is_first = false; + } else { // if an item in a flow sequence + strcpy(buff, token.data.scalar.value); + strcat(my_files.files[j].keys[i].value, " "); + strcat(my_files.files[j].keys[i].value, buff); + } + } else {strcpy(key_value,token.data.scalar.value);} diff --git a/test_fms/diag_manager/test_cell_measures.sh b/test_fms/diag_manager/test_cell_measures.sh index c97216fdb..4d05f6831 100755 --- a/test_fms/diag_manager/test_cell_measures.sh +++ b/test_fms/diag_manager/test_cell_measures.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: static_file diff --git a/test_fms/diag_manager/test_diag_manager2.sh b/test_fms/diag_manager/test_diag_manager2.sh index 7b280c885..5f83687a4 100755 --- a/test_fms/diag_manager/test_diag_manager2.sh +++ b/test_fms/diag_manager/test_diag_manager2.sh @@ -514,7 +514,7 @@ if [ -z "${skipflag}" ]; then cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [ 2, 1, 1, 0, 0, 0 ] diag_files: - file_name: wild_card_name%4yr%2mo%2dy%2hr filename_time: end @@ -592,7 +592,7 @@ my_test_count=`expr $my_test_count + 1` cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0 ] diag_files: - file_name: wild_card_name%4yr%2mo%2dy%2hr filename_time: end @@ -673,7 +673,7 @@ _EOF printf "&diag_manager_nml \n use_modern_diag = .true. \n/" | cat > input.nml cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0 ] diag_files: - file_name: file1 freq: 6 hours @@ -722,7 +722,7 @@ _EOF printf "&diag_manager_nml \n use_modern_diag = .true. \n do_diag_field_log = .true. \n/" | cat > input.nml cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0 ] diag_files: - file_name: static_file @@ -886,7 +886,7 @@ _EOF cat <<_EOF > diag_out_ref.yaml --- title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: static_file freq: -1 @@ -1296,7 +1296,7 @@ test_expect_success "check modern diag manager yaml output (test $my_test_count) printf "&diag_manager_nml \n use_modern_diag = .true. \n use_clock_average = .true. \n /" | cat > input.nml cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0 ] diag_files: - file_name: file1_clock @@ -1318,7 +1318,7 @@ my_test_count=`expr $my_test_count + 1` printf "&diag_manager_nml \n use_modern_diag = .true. \n use_clock_average = .false. \n /" | cat > input.nml cat <<_EOF > diag_table.yaml title: test_diag_manager -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0 ] diag_files: - file_name: file1_forecast diff --git a/test_fms/diag_manager/test_dm_weights.sh b/test_fms/diag_manager/test_dm_weights.sh index 0f291180a..a4b324899 100755 --- a/test_fms/diag_manager/test_dm_weights.sh +++ b/test_fms/diag_manager/test_dm_weights.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_weights -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_weights time_units: hours diff --git a/test_fms/diag_manager/test_flush_nc_file.sh b/test_fms/diag_manager/test_flush_nc_file.sh index 7337bd849..4b31d2054 100755 --- a/test_fms/diag_manager/test_flush_nc_file.sh +++ b/test_fms/diag_manager/test_flush_nc_file.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_nc_flush -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_flush time_units: hours diff --git a/test_fms/diag_manager/test_multiple_send_data.sh b/test_fms/diag_manager/test_multiple_send_data.sh index 1240e1d41..f9b54ac8a 100755 --- a/test_fms/diag_manager/test_multiple_send_data.sh +++ b/test_fms/diag_manager/test_multiple_send_data.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_multiple_sends -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_multiple_sends time_units: hours diff --git a/test_fms/diag_manager/test_output_every_freq.sh b/test_fms/diag_manager/test_output_every_freq.sh index 71f6cad23..60fd82a1a 100755 --- a/test_fms/diag_manager/test_output_every_freq.sh +++ b/test_fms/diag_manager/test_output_every_freq.sh @@ -30,7 +30,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_diag_manager_01 -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_0days time_units: days diff --git a/test_fms/diag_manager/test_prepend_date.sh b/test_fms/diag_manager/test_prepend_date.sh index 13bbf7c77..17930b0eb 100755 --- a/test_fms/diag_manager/test_prepend_date.sh +++ b/test_fms/diag_manager/test_prepend_date.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_prepend_date -base_date: 1 1 1 0 0 0 +base_date: [1, 1, 1, 0, 0, 0] diag_files: - file_name: test_non_static time_units: hours diff --git a/test_fms/diag_manager/test_subregional.sh b/test_fms/diag_manager/test_subregional.sh index dcb1f5e9d..4e36f282e 100755 --- a/test_fms/diag_manager/test_subregional.sh +++ b/test_fms/diag_manager/test_subregional.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_subregional -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: # This is to test a file with multiple z axis diff --git a/test_fms/diag_manager/test_time_avg.sh b/test_fms/diag_manager/test_time_avg.sh index e31e243fa..0abead921 100755 --- a/test_fms/diag_manager/test_time_avg.sh +++ b/test_fms/diag_manager/test_time_avg.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_avg -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_avg time_units: hours diff --git a/test_fms/diag_manager/test_time_diurnal.sh b/test_fms/diag_manager/test_time_diurnal.sh index aef8a29e0..30897cf00 100755 --- a/test_fms/diag_manager/test_time_diurnal.sh +++ b/test_fms/diag_manager/test_time_diurnal.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_diurnal -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_diurnal time_units: hours diff --git a/test_fms/diag_manager/test_time_max.sh b/test_fms/diag_manager/test_time_max.sh index d2a0fd7cd..e47b43de5 100755 --- a/test_fms/diag_manager/test_time_max.sh +++ b/test_fms/diag_manager/test_time_max.sh @@ -31,7 +31,7 @@ output_dir #TODO replace with yaml diag_table and set diag_manager_nml::use_modern_diag=.true. cat <<_EOF > diag_table.yaml title: test_max -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_max time_units: hours diff --git a/test_fms/diag_manager/test_time_min.sh b/test_fms/diag_manager/test_time_min.sh index f2969d47c..ed9e35581 100755 --- a/test_fms/diag_manager/test_time_min.sh +++ b/test_fms/diag_manager/test_time_min.sh @@ -31,7 +31,7 @@ output_dir #TODO replace with yaml diag_table and set diag_manager_nml::use_modern_diag=.true. cat <<_EOF > diag_table.yaml title: test_min -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_min time_units: hours diff --git a/test_fms/diag_manager/test_time_none.sh b/test_fms/diag_manager/test_time_none.sh index 502a7b84b..e5057e442 100755 --- a/test_fms/diag_manager/test_time_none.sh +++ b/test_fms/diag_manager/test_time_none.sh @@ -30,7 +30,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_none -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_none freq: 6 hours diff --git a/test_fms/diag_manager/test_time_pow.sh b/test_fms/diag_manager/test_time_pow.sh index 5e343de9b..53231f7a9 100755 --- a/test_fms/diag_manager/test_time_pow.sh +++ b/test_fms/diag_manager/test_time_pow.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_pow -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_pow time_units: hours diff --git a/test_fms/diag_manager/test_time_rms.sh b/test_fms/diag_manager/test_time_rms.sh index 8f3c526f7..86dd98889 100755 --- a/test_fms/diag_manager/test_time_rms.sh +++ b/test_fms/diag_manager/test_time_rms.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_rms -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_rms time_units: hours diff --git a/test_fms/diag_manager/test_time_sum.sh b/test_fms/diag_manager/test_time_sum.sh index c7631217a..639ba930f 100755 --- a/test_fms/diag_manager/test_time_sum.sh +++ b/test_fms/diag_manager/test_time_sum.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_sum -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_sum time_units: hours diff --git a/test_fms/diag_manager/test_var_masks.sh b/test_fms/diag_manager/test_var_masks.sh index 761fb345c..81151492b 100755 --- a/test_fms/diag_manager/test_var_masks.sh +++ b/test_fms/diag_manager/test_var_masks.sh @@ -28,7 +28,7 @@ output_dir cat <<_EOF > diag_table.yaml title: test_var_masks -base_date: 2 1 1 0 0 0 +base_date: [2, 1, 1, 0, 0, 0] diag_files: - file_name: test_var_masks freq: 1 days