From c7175a40437a179ce7f149029e7c09be38b184dd Mon Sep 17 00:00:00 2001 From: Edward Hartnett <38856240+edwardhartnett@users.noreply.github.com> Date: Wed, 28 Feb 2024 05:39:06 -0700 Subject: [PATCH] more large file testing (#484) * more testing of large files * fixing large file test * running more large file tests * changed name of util file * more changes * more progress --- CMakeLists.txt | 2 +- src/g2cfile.c | 2 + src/g2cindex.c | 2 +- tests/CMakeLists.txt | 14 +++-- ...swave.t00z.wcoast.0p16.f000.grib2.degrib2} | 0 tests/{util.c => g2c_test_util.c} | 6 +- tests/run_degrib2_tests.sh | 2 +- tests/run_extra_index_tests.sh | 25 ++++++++ tests/run_index_tests.sh | 2 +- tests/run_large_index_tests.sh | 2 +- tests/tst_degrib2.c | 2 +- tests/tst_degrib2_fast.c | 2 +- tests/tst_degrib2_ftp_all.c | 63 +++++++++++++------ 13 files changed, 91 insertions(+), 33 deletions(-) rename tests/data/{ref_gdaswave.degrib2.txt => ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2} (100%) rename tests/{util.c => g2c_test_util.c} (96%) create mode 100644 tests/run_extra_index_tests.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 20a1ff29..45d67ad4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,7 @@ option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(BUILD_STATIC_LIBS "Build static libraries" ON) option(FTP_TEST_FILES "Fetch and test with files on FTP site." OFF) option(FTP_LARGE_TEST_FILES "Fetch and test with very large files on FTP site." OFF) -option(FTP_EXTRA_TEST_FILES "Test with files not yet available via FTP." OFF) +option(FTP_EXTRA_TEST_FILES "Test with more large files fetched via FTP." OFF) option(LOGGING "Turn on internal logging messages. Only useful to g2c developers." OFF) option(PTHREADS "Turn on thread-safty with pthreads." OFF) option(UTILS "Build and install some utility programs." ON) diff --git a/src/g2cfile.c b/src/g2cfile.c index 262edc45..8b55a385 100644 --- a/src/g2cfile.c +++ b/src/g2cfile.c @@ -1194,6 +1194,8 @@ g2c_open(const char *path, int mode, int *g2cid) { int ret; + LOG((2, "g2c_open path %s mode %d", path, mode)); + /* If using threading, lock the mutex. */ MUTEX_LOCK(m); diff --git a/src/g2cindex.c b/src/g2cindex.c index 6174c145..a96aacd0 100644 --- a/src/g2cindex.c +++ b/src/g2cindex.c @@ -465,7 +465,7 @@ g2c_write_index(int g2cid, int mode, const char *index_file) } } - /* If LARGE_INDEX_FILE, check if file exists. */ + /* If LARGE_INDEX_FILE, set index version. */ if (mode & G2C_LARGE_FILE_INDEX) index_version = 2; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b80ccc5b..84a4ef74 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,7 +51,7 @@ set(REF_FILES "gdaswave.t00z.wcoast.0p16.f000.grib2.idx" "ref_gdas.t12z.pgrb2.1p00.anl.degrib2" "ref_gdas.t12z.pgrb2.1p00.anl.grib2.degrib2" "ref_gdas.t12z.pgrb2.1p00.anl.grib2.grb2index" - "ref_gdaswave.degrib2.txt" + "ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2" "ref_gdaswave.t00z.wcoast.0p16.f000.grb2index" "ref_GLOBAL.grib2.2022103000.0000.degrib2" "ref_GLOBAL.grib2.2022103000.0000.grb2index" @@ -74,7 +74,7 @@ set(REF_FILES "gdaswave.t00z.wcoast.0p16.f000.grib2.idx" "ref_tst_jasper_warning.txt" ) -# Copy extra files if needed. +# Copy large files if needed. if(FTP_LARGE_TEST_FILES) set(REF_FILES ${REF_FILES} "ref_fv3lam.t00z.prslev.f000.grib2.degrib2") endif() @@ -94,7 +94,7 @@ endif() # Build a C program test. function(g2c_build_test name) - add_executable(${name} ${name}.c util.c) + add_executable(${name} ${name}.c g2c_test_util.c) target_link_libraries(${name} PUBLIC g2c::g2c) if(BUILD_G2C) target_link_libraries(${name} PRIVATE ${LIBXML2_LIBRARIES}) @@ -104,7 +104,7 @@ endfunction() # Build a C program test. function(g2c_build_test name) - add_executable(${name} ${name}.c util.c) + add_executable(${name} ${name}.c g2c_test_util.c) target_link_libraries(${name} PUBLIC g2c::g2c) target_link_libraries(${name} PRIVATE ${LIBXML2_LIBRARIES}) target_include_directories(${name} PRIVATE "${LIBXML2_INCLUDE_DIR}") @@ -164,7 +164,8 @@ if(FTP_TEST_FILES) endif() if(FTP_EXTRA_TEST_FILES) - set(EXTRA_FTP_FILES "MRMS_MultiSensor_QPE_24H_Pass2_00.00_20230621-110000.grib2") + set(EXTRA_FTP_FILES MRMS_MultiSensor_QPE_24H_Pass2_00.00_20230621-110000.grib2 + GFSPRS.GrbF06 rrfs.t18z.prslev.f000.grib2) foreach(THE_FILE IN LISTS EXTRA_FTP_FILES) PULL_DATA(${G2_FTP_URL} ${THE_FILE}) endforeach() @@ -253,6 +254,9 @@ if(BUILD_G2C) if(FTP_LARGE_TEST_FILES) gu_test(run_large_index_tests) endif() + if(FTP_EXTRA_TEST_FILES) + gu_test(run_extra_index_tests) + endif() endif() endif() endif() diff --git a/tests/data/ref_gdaswave.degrib2.txt b/tests/data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 similarity index 100% rename from tests/data/ref_gdaswave.degrib2.txt rename to tests/data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 diff --git a/tests/util.c b/tests/g2c_test_util.c similarity index 96% rename from tests/util.c rename to tests/g2c_test_util.c index 2a106039..ad917fba 100644 --- a/tests/util.c +++ b/tests/g2c_test_util.c @@ -41,7 +41,7 @@ cmpString(const void *p, const void *q) * Ed Hartnett 10/6/22 */ int -degrib2_lines_not_equal(char *line1, char *l2) +degrib2_lines_not_equal(int cnt, char *line1, char *l2) { char long_abbrev[MAX_LINE_LEN + 1], long_cmin[MAX_LINE_LEN + 1]; char long_cavg[MAX_LINE_LEN + 1], long_cmax[MAX_LINE_LEN + 1]; @@ -110,6 +110,7 @@ degrib2_lines_not_equal(char *line1, char *l2) } else { + printf("\nline %d\n", cnt); printf("\n%s\n", line1); printf("expected:\n%s\n", l2); return G2C_ERROR; @@ -127,6 +128,7 @@ compare_degrib2_files2(char *fname1, char *fname2) { FILE *fp1, *fp2; char l1[MAX_LINE_LEN + 1], l2[MAX_LINE_LEN + 1]; + int cnt = 1; /* Open the two files. */ if (!(fp1 = fopen(fname1, "r"))) @@ -141,7 +143,7 @@ compare_degrib2_files2(char *fname1, char *fname2) return G2C_ERROR; /* printf("l1: %s\n", l1); */ /* printf("l2: %s\n", l2); */ - if (degrib2_lines_not_equal(l1, l2)) + if (degrib2_lines_not_equal(cnt++, l1, l2)) return G2C_ERROR; } diff --git a/tests/run_degrib2_tests.sh b/tests/run_degrib2_tests.sh index e5ab3760..5f4a8736 100644 --- a/tests/run_degrib2_tests.sh +++ b/tests/run_degrib2_tests.sh @@ -13,7 +13,7 @@ echo "*** Running g2c_degrib2 test" ../utils/g2c_degrib2 -v -o g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/gdaswave.t00z.wcoast.0p16.f000.grib2 # Check against expected output. -diff -w g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/ref_gdaswave.degrib2.txt +diff -w g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 echo "*** SUCCESS!" exit 0 diff --git a/tests/run_extra_index_tests.sh b/tests/run_extra_index_tests.sh new file mode 100644 index 00000000..c56cbd60 --- /dev/null +++ b/tests/run_extra_index_tests.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# This is a test script for the NCEPLIBS-g2c project. +# +# This script tests the g2c_index utility on very large files, which +# are only fetch from FTP if the CMake build includes the +# FTP_EXTRA_TEST_FILES option ON. +# +# Ed Hartnett, 2/22/24 + +set -e +echo "" +echo "*** Running g2c_index extra file test" + +# rrfs.t18z.prslev.f000.grib2 +# Create an index for a GRIB2 file. +../utils/g2c_index -o GFSPRS.GrbF06.idx -l -v data/GFSPRS.GrbF06 + +# Summarize the index data. +#../utils/g2c_degrib2 -v -o fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/fv3lam.t00z.prslev.f000.grib2 fv3lam.t00z.prslev.f000.grib2.idx + +# Check against expected output. +#diff -w fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/ref_fv3lam.t00z.prslev.f000.grib2.degrib2 + +echo "*** SUCCESS!" +exit 0 diff --git a/tests/run_index_tests.sh b/tests/run_index_tests.sh index 7c10a68d..0b9e37ab 100644 --- a/tests/run_index_tests.sh +++ b/tests/run_index_tests.sh @@ -16,7 +16,7 @@ echo "*** Running g2c_index test" ../utils/g2c_degrib2 -v -o gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 data/gdaswave.t00z.wcoast.0p16.f000.grib2 gdaswave.t00z.wcoast.0p16.f000.grib2.idx # Check against expected output. -diff -w gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 data/ref_gdaswave.degrib2.txt +diff -w gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 echo "*** SUCCESS!" exit 0 diff --git a/tests/run_large_index_tests.sh b/tests/run_large_index_tests.sh index 9445a8a5..2294c4e5 100644 --- a/tests/run_large_index_tests.sh +++ b/tests/run_large_index_tests.sh @@ -13,7 +13,7 @@ echo "*** Running g2c_index large file test" ../utils/g2c_index -o fv3lam.t00z.prslev.f000.grib2.idx -l -v data/fv3lam.t00z.prslev.f000.grib2 # Summarize the index data. -../utils/g2c_degrib2 -v -o fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/fv3lam.t00z.prslev.f000.grib2 fv3lam.t00z.prslev.f000.grib2.idx +../utils/g2c_degrib2 -o fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/fv3lam.t00z.prslev.f000.grib2 fv3lam.t00z.prslev.f000.grib2.idx # Check against expected output. #diff -w fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/ref_fv3lam.t00z.prslev.f000.grib2.degrib2 diff --git a/tests/tst_degrib2.c b/tests/tst_degrib2.c index d50e0321..d5b75124 100644 --- a/tests/tst_degrib2.c +++ b/tests/tst_degrib2.c @@ -11,7 +11,7 @@ #define FILE_NAME "tst_degrib2.txt" #define WAVE_FILE "data/gdaswave.t00z.wcoast.0p16.f000.grib2" -#define REF_FILE "data/ref_gdaswave.degrib2.txt" +#define REF_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2" #define DEGRIB2_FILE "gdaswave.t00z.wcoast.0p16.f000.degrib2" #define REF_INDEX_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grb2index" #define TEST_INDEX_FILE "tst_gdaswave.t00z.wcoast.0p16.f000.grb2index" diff --git a/tests/tst_degrib2_fast.c b/tests/tst_degrib2_fast.c index fc9d3840..309e47ee 100644 --- a/tests/tst_degrib2_fast.c +++ b/tests/tst_degrib2_fast.c @@ -11,7 +11,7 @@ #define FILE_NAME "tst_degrib2.txt" #define WAVE_FILE "data/gdaswave.t00z.wcoast.0p16.f000.grib2" -#define REF_FILE "data/ref_gdaswave.degrib2.txt" +#define REF_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2" #define DEGRIB2_FILE "gdaswave.t00z.wcoast.0p16.f000.degrib2" #define REF_INDEX_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grb2index" #define TEST_INDEX_FILE "tst_gdaswave.t00z.wcoast.0p16.f000.grb2index" diff --git a/tests/tst_degrib2_ftp_all.c b/tests/tst_degrib2_ftp_all.c index 64d6a302..bd7ea929 100644 --- a/tests/tst_degrib2_ftp_all.c +++ b/tests/tst_degrib2_ftp_all.c @@ -19,24 +19,36 @@ main() #ifdef FTP_TEST_FILES printf("Testing degrib2 on files downloaded via FTP...\n"); { -#define NUM_FILES 10 +#define NUM_FILES 11 char file[NUM_FILES][MAX_FILENAME_LEN + 1] = { + "data/seaice.t00z.grb.grib2", "data/gdaswave.t00z.wcoast.0p16.f000.grib2", "data/blend.t19z.core.f001.co.grib2", "data/cmc_geavg.t12z.pgrb2a.0p50.f000", "data/flxf2022111712.01.2022111712.grb2", "data/GLOBAL.grib2.2022103000.0000", "data/hiresw.t00z.arw_5km.f00.hi.grib2", "data/naefs_ge10pt.t12z.pgrb2a.0p50_bcf003", - "data/rap.t00z.awp130pgrbf00.grib2", "data/seaice.t00z.grb.grib2", + "data/rap.t00z.awp130pgrbf00.grib2", "data/sgx_nwps_CG3_20221117_1200.grib2", "data/aqm.t12z.max_8hr_o3.227.grib2" }; + char index_file[NUM_FILES][MAX_FILENAME_LEN + 1] = { + "seaice.t00z.grb.grib2.idx", "gdaswave.t00z.wcoast.0p16.f000.grib2.idx", + "blend.t19z.core.f001.co.grib2.idx", "cmc_geavg.t12z.pgrb2a.0p50.f000.idx", + "flxf2022111712.01.2022111712.grb2.idx", "GLOBAL.grib2.2022103000.0000.idx", + "hiresw.t00z.arw_5km.f00.hi.grib2.idx", "naefs_ge10pt.t12z.pgrb2a.0p50_bcf003.idx", + "rap.t00z.awp130pgrbf00.grib2.idx", + "sgx_nwps_CG3_20221117_1200.grib2.idx", "aqm.t12z.max_8hr_o3.227.grib2.idx" + }; + int expected_nmsg[NUM_FILES] = {1, 19, 72, 72, 99, 219, 239, 49, 306, 1305, 3}; int g2cid; int num_msg; int f; int t; int ret; + g2c_set_log_level(15); /* for (f = 0; f < NUM_FILES; f++) */ - for (f = 0; f < 9; f++) + for (f = 0; f < 1; f++) { - for (t = 0; t < 1; t++) + /* for (t = 0; t < 2; t++) */ + for (t = 0; t < 2; t++) { char degrib2_file[MAX_FILENAME_LEN + 9]; char ref_degrib2_file[MAX_FILENAME_LEN + 20]; @@ -44,24 +56,29 @@ main() /* Open the data file with and without the index file. */ if (t) { - /* printf("\ttesting degrib2 on file %s downloaded via FTP using index...", file[f]); */ - /* if ((ret = g2c_open_index(file[f], REF_GDAS_INDEX_FILE, 0, &g2cid))) */ - /* return ret; */ + printf("\ttesting degrib2 on file %s downloaded via FTP using index...", file[f]); + if ((ret = g2c_open_index(file[f], index_file[f], 0, &g2cid))) + return ret; } else { printf("\ttesting degrib2 on file %s downloaded via FTP without using index...", file[f]); if ((ret = g2c_open(file[f], 0, &g2cid))) return ret; + if ((ret = g2c_write_index(g2cid, 0, index_file[f]))) + return ret; } /* Check some stuff. */ if ((ret = g2c_inq(g2cid, &num_msg))) return ret; - printf("num_msg %d\n", num_msg); + if (num_msg != expected_nmsg[f]) + { + printf("num_msg %d expected %d\n", num_msg, expected_nmsg[f]); + return G2C_ERROR; + } /* Output a degrib2 file. */ - /* g2c_set_log_level(15); */ sprintf(degrib2_file, "%s.degrib2", basename(file[f])); if ((ret = g2c_degrib2(g2cid, degrib2_file))) return ret; @@ -75,20 +92,24 @@ main() if ((ret = compare_degrib2_files2(degrib2_file, ref_degrib2_file))) { printf("compare failed!\n"); - return ret; + /*return ret;*/ } + printf("\tok!\n"); } - printf("\tok!\n"); } } printf("ok!\n"); -#ifdef LARGE_FTP_TESTS +#ifdef LARGE_FTP_TESTS1 printf("Testing degrib2 on very large files downloaded via FTP...\n"); { #define NUM_LARGE_FILES 1 +#define FV3_NUM_MSG 1081 char file[NUM_LARGE_FILES][MAX_FILENAME_LEN + 1] = { "data/fv3lam.t00z.prslev.f000.grib2" }; + char index_file[NUM_LARGE_FILES][MAX_FILENAME_LEN + 1] = { + "fv3lam.t00z.prslev.f000.grib2.idx" + }; int g2cid; int num_msg; int f; @@ -98,8 +119,7 @@ main() /* g2c_set_log_level(10); */ for (f = 0; f < NUM_LARGE_FILES; f++) { - /* for (t = 0; t < 2; t++) */ - for (t = 0; t < 1; t++) + for (t = 0; t < 2; t++) { char degrib2_file[MAX_FILENAME_LEN + 9]; char ref_degrib2_file[MAX_FILENAME_LEN + 20]; @@ -107,23 +127,28 @@ main() /* Open the data file with and without the index file. */ if (t) { + /* Open the data with the index file. */ printf("\ttesting degrib2 on file %s downloaded via FTP using index...", file[f]); - /* if ((ret = g2c_open_index(file[f], REF_GDAS_INDEX_FILE, 0, &g2cid))) */ - /* return ret; */ + if ((ret = g2c_open_index(file[f], index_file[f], 0, &g2cid))) + return ret; } else { + /* Open the data and create the index file. */ printf("\ttesting degrib2 on file %s downloaded via FTP without using index...", file[f]); if ((ret = g2c_open(file[f], 0, &g2cid))) return ret; - } + + if ((ret = g2c_write_index(g2cid, 0, index_file[f]))) + return ret; + } /* Check some stuff. */ if ((ret = g2c_inq(g2cid, &num_msg))) return ret; printf("num_msg %d\n", num_msg); - /* if (num_msg != GDAS_NUM_MSG) */ - /* return G2C_ERROR; */ + if (num_msg != FV3_NUM_MSG) + return G2C_ERROR; /* Output a degrib2 file. */ sprintf(degrib2_file, "%s.degrib2", basename(file[f]));