more large file testing (#484)

* more testing of large files

* fixing large file test

* running more large file tests

* changed name of util file

* more changes

* more progress

CMakeLists.txt

@@ -25,7 +25,7 @@ option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 option(BUILD_STATIC_LIBS "Build static libraries" ON)
 option(FTP_TEST_FILES "Fetch and test with files on FTP site." OFF)
 option(FTP_LARGE_TEST_FILES "Fetch and test with very large files on FTP site." OFF)
-option(FTP_EXTRA_TEST_FILES "Test with files not yet available via FTP." OFF)
+option(FTP_EXTRA_TEST_FILES "Test with more large files fetched via FTP." OFF)
 option(LOGGING "Turn on internal logging messages. Only useful to g2c developers." OFF)
 option(PTHREADS "Turn on thread-safty with pthreads." OFF)
 option(UTILS "Build and install some utility programs." ON)

src/g2cfile.c

@@ -1194,6 +1194,8 @@ g2c_open(const char *path, int mode, int *g2cid)
 {
     int ret;
 
+    LOG((2, "g2c_open path %s mode %d", path, mode));
+
     /* If using threading, lock the mutex. */
     MUTEX_LOCK(m);
 

src/g2cindex.c

@@ -465,7 +465,7 @@ g2c_write_index(int g2cid, int mode, const char *index_file)
         }
     }
 
-    /* If LARGE_INDEX_FILE, check if file exists. */
+    /* If LARGE_INDEX_FILE, set index version. */
     if (mode & G2C_LARGE_FILE_INDEX)
 	index_version = 2;
 

tests/CMakeLists.txt

@@ -51,7 +51,7 @@ set(REF_FILES "gdaswave.t00z.wcoast.0p16.f000.grib2.idx"
   "ref_gdas.t12z.pgrb2.1p00.anl.degrib2"
   "ref_gdas.t12z.pgrb2.1p00.anl.grib2.degrib2"
   "ref_gdas.t12z.pgrb2.1p00.anl.grib2.grb2index"
-  "ref_gdaswave.degrib2.txt"
+  "ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2"
   "ref_gdaswave.t00z.wcoast.0p16.f000.grb2index"
   "ref_GLOBAL.grib2.2022103000.0000.degrib2"
   "ref_GLOBAL.grib2.2022103000.0000.grb2index"
@@ -74,7 +74,7 @@ set(REF_FILES "gdaswave.t00z.wcoast.0p16.f000.grib2.idx"
   "ref_tst_jasper_warning.txt"
   )
 
-# Copy extra files if needed.
+# Copy large files if needed.
 if(FTP_LARGE_TEST_FILES)
   set(REF_FILES ${REF_FILES} "ref_fv3lam.t00z.prslev.f000.grib2.degrib2")
 endif()
@@ -94,7 +94,7 @@ endif()
 
 # Build a C program test.
 function(g2c_build_test name)
-  add_executable(${name} ${name}.c util.c)
+  add_executable(${name} ${name}.c g2c_test_util.c)
   target_link_libraries(${name} PUBLIC g2c::g2c)
   if(BUILD_G2C)
     target_link_libraries(${name} PRIVATE ${LIBXML2_LIBRARIES})
@@ -104,7 +104,7 @@ endfunction()
 
 # Build a C program test.
 function(g2c_build_test name)
-  add_executable(${name} ${name}.c util.c)
+  add_executable(${name} ${name}.c g2c_test_util.c)
   target_link_libraries(${name} PUBLIC g2c::g2c)
   target_link_libraries(${name} PRIVATE ${LIBXML2_LIBRARIES})
   target_include_directories(${name} PRIVATE "${LIBXML2_INCLUDE_DIR}")
@@ -164,7 +164,8 @@ if(FTP_TEST_FILES)
   endif()
 
   if(FTP_EXTRA_TEST_FILES)
-    set(EXTRA_FTP_FILES "MRMS_MultiSensor_QPE_24H_Pass2_00.00_20230621-110000.grib2")
+    set(EXTRA_FTP_FILES MRMS_MultiSensor_QPE_24H_Pass2_00.00_20230621-110000.grib2
+      GFSPRS.GrbF06 rrfs.t18z.prslev.f000.grib2)
     foreach(THE_FILE IN LISTS EXTRA_FTP_FILES) 
       PULL_DATA(${G2_FTP_URL} ${THE_FILE})
     endforeach()
@@ -253,6 +254,9 @@ if(BUILD_G2C)
       if(FTP_LARGE_TEST_FILES)
 	gu_test(run_large_index_tests)
       endif()
+      if(FTP_EXTRA_TEST_FILES)
+	gu_test(run_extra_index_tests)
+      endif()
     endif()
   endif()
 endif()

tests/util.c → tests/g2c_test_util.c

@@ -41,7 +41,7 @@ cmpString(const void *p, const void *q)
  * Ed Hartnett 10/6/22
  */
 int
-degrib2_lines_not_equal(char *line1, char *l2)
+degrib2_lines_not_equal(int cnt, char *line1, char *l2)
 {
     char long_abbrev[MAX_LINE_LEN + 1], long_cmin[MAX_LINE_LEN + 1];
     char long_cavg[MAX_LINE_LEN + 1], long_cmax[MAX_LINE_LEN + 1];
@@ -110,6 +110,7 @@ degrib2_lines_not_equal(char *line1, char *l2)
     }
     else
     {
+        printf("\nline %d\n", cnt);
         printf("\n%s\n", line1);
         printf("expected:\n%s\n", l2);
         return G2C_ERROR;
@@ -127,6 +128,7 @@ compare_degrib2_files2(char *fname1, char *fname2)
 {
     FILE *fp1, *fp2;
     char l1[MAX_LINE_LEN + 1], l2[MAX_LINE_LEN + 1];
+    int cnt = 1;
 
     /* Open the two files. */
     if (!(fp1 = fopen(fname1, "r")))
@@ -141,7 +143,7 @@ compare_degrib2_files2(char *fname1, char *fname2)
             return G2C_ERROR;
         /* printf("l1: %s\n", l1); */
         /* printf("l2: %s\n", l2); */
-        if (degrib2_lines_not_equal(l1, l2))
+        if (degrib2_lines_not_equal(cnt++, l1, l2))
             return G2C_ERROR;
     }
 

tests/run_degrib2_tests.sh

@@ -13,7 +13,7 @@ echo "*** Running g2c_degrib2 test"
 ../utils/g2c_degrib2 -v -o g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/gdaswave.t00z.wcoast.0p16.f000.grib2 
 
 # Check against expected output.
-diff -w g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/ref_gdaswave.degrib2.txt
+diff -w g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 
 
 echo "*** SUCCESS!"
 exit 0

tests/run_extra_index_tests.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+# This is a test script for the NCEPLIBS-g2c project.
+#
+# This script tests the g2c_index utility on very large files, which
+# are only fetch from FTP if the CMake build includes the
+# FTP_EXTRA_TEST_FILES option ON.
+#
+# Ed Hartnett, 2/22/24
+
+set -e
+echo ""
+echo "*** Running g2c_index extra file test"
+
+# rrfs.t18z.prslev.f000.grib2
+# Create an index for a GRIB2 file.
+../utils/g2c_index -o GFSPRS.GrbF06.idx -l -v data/GFSPRS.GrbF06
+
+# Summarize the index data.
+#../utils/g2c_degrib2 -v -o fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/fv3lam.t00z.prslev.f000.grib2 fv3lam.t00z.prslev.f000.grib2.idx 
+
+# Check against expected output.
+#diff -w fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/ref_fv3lam.t00z.prslev.f000.grib2.degrib2
+
+echo "*** SUCCESS!"
+exit 0

tests/run_index_tests.sh

@@ -16,7 +16,7 @@ echo "*** Running g2c_index test"
 ../utils/g2c_degrib2 -v -o gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 data/gdaswave.t00z.wcoast.0p16.f000.grib2 gdaswave.t00z.wcoast.0p16.f000.grib2.idx 
 
 # Check against expected output.
-diff -w gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 data/ref_gdaswave.degrib2.txt
+diff -w gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2
 
 echo "*** SUCCESS!"
 exit 0

tests/run_large_index_tests.sh

@@ -13,7 +13,7 @@ echo "*** Running g2c_index large file test"
 ../utils/g2c_index -o fv3lam.t00z.prslev.f000.grib2.idx -l -v data/fv3lam.t00z.prslev.f000.grib2 
 
 # Summarize the index data.
-../utils/g2c_degrib2 -v -o fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/fv3lam.t00z.prslev.f000.grib2 fv3lam.t00z.prslev.f000.grib2.idx 
+../utils/g2c_degrib2 -o fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/fv3lam.t00z.prslev.f000.grib2 fv3lam.t00z.prslev.f000.grib2.idx 
 
 # Check against expected output.
 #diff -w fv3lam.t00z.prslev.f000.grib2.idx.degrib2 data/ref_fv3lam.t00z.prslev.f000.grib2.degrib2

tests/tst_degrib2.c

@@ -11,7 +11,7 @@
 
 #define FILE_NAME "tst_degrib2.txt"
 #define WAVE_FILE "data/gdaswave.t00z.wcoast.0p16.f000.grib2"
-#define REF_FILE "data/ref_gdaswave.degrib2.txt"
+#define REF_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2"
 #define DEGRIB2_FILE "gdaswave.t00z.wcoast.0p16.f000.degrib2"
 #define REF_INDEX_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grb2index"
 #define TEST_INDEX_FILE "tst_gdaswave.t00z.wcoast.0p16.f000.grb2index"

tests/tst_degrib2_fast.c

@@ -11,7 +11,7 @@
 
 #define FILE_NAME "tst_degrib2.txt"
 #define WAVE_FILE "data/gdaswave.t00z.wcoast.0p16.f000.grib2"
-#define REF_FILE "data/ref_gdaswave.degrib2.txt"
+#define REF_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2"
 #define DEGRIB2_FILE "gdaswave.t00z.wcoast.0p16.f000.degrib2"
 #define REF_INDEX_FILE "data/ref_gdaswave.t00z.wcoast.0p16.f000.grb2index"
 #define TEST_INDEX_FILE "tst_gdaswave.t00z.wcoast.0p16.f000.grb2index"

tests/tst_degrib2_ftp_all.c

@@ -19,49 +19,66 @@ main()
 #ifdef FTP_TEST_FILES
     printf("Testing degrib2 on files downloaded via FTP...\n");
     {
-#define NUM_FILES 10
+#define NUM_FILES 11
         char file[NUM_FILES][MAX_FILENAME_LEN + 1] = {
+	    "data/seaice.t00z.grb.grib2", "data/gdaswave.t00z.wcoast.0p16.f000.grib2",
             "data/blend.t19z.core.f001.co.grib2", "data/cmc_geavg.t12z.pgrb2a.0p50.f000",
             "data/flxf2022111712.01.2022111712.grb2", "data/GLOBAL.grib2.2022103000.0000",
             "data/hiresw.t00z.arw_5km.f00.hi.grib2", "data/naefs_ge10pt.t12z.pgrb2a.0p50_bcf003",
-            "data/rap.t00z.awp130pgrbf00.grib2", "data/seaice.t00z.grb.grib2",
+            "data/rap.t00z.awp130pgrbf00.grib2", 
             "data/sgx_nwps_CG3_20221117_1200.grib2", "data/aqm.t12z.max_8hr_o3.227.grib2"
         };
+        char index_file[NUM_FILES][MAX_FILENAME_LEN + 1] = {
+	    "seaice.t00z.grb.grib2.idx", "gdaswave.t00z.wcoast.0p16.f000.grib2.idx",
+            "blend.t19z.core.f001.co.grib2.idx", "cmc_geavg.t12z.pgrb2a.0p50.f000.idx",
+            "flxf2022111712.01.2022111712.grb2.idx", "GLOBAL.grib2.2022103000.0000.idx",
+            "hiresw.t00z.arw_5km.f00.hi.grib2.idx", "naefs_ge10pt.t12z.pgrb2a.0p50_bcf003.idx",
+            "rap.t00z.awp130pgrbf00.grib2.idx", 
+            "sgx_nwps_CG3_20221117_1200.grib2.idx", "aqm.t12z.max_8hr_o3.227.grib2.idx"
+        };
+	int expected_nmsg[NUM_FILES] = {1, 19, 72, 72, 99, 219, 239, 49, 306, 1305, 3};
         int g2cid;
         int num_msg;
         int f;
         int t;
         int ret;
+	g2c_set_log_level(15);
 
         /* for (f = 0; f < NUM_FILES; f++) */
-        for (f = 0; f < 9; f++)
+        for (f = 0; f < 1; f++)
         {
-            for (t = 0; t < 1; t++)
+            /* for (t = 0; t < 2; t++) */
+            for (t = 0; t < 2; t++)
             {
                 char degrib2_file[MAX_FILENAME_LEN + 9];
                 char ref_degrib2_file[MAX_FILENAME_LEN + 20];
 
                 /* Open the data file with and without the index file. */
                 if (t)
                 {
-                    /* printf("\ttesting degrib2 on file %s downloaded via FTP using index...", file[f]); */
-                    /* if ((ret = g2c_open_index(file[f], REF_GDAS_INDEX_FILE, 0, &g2cid))) */
-                    /*     return ret; */
+                    printf("\ttesting degrib2 on file %s downloaded via FTP using index...", file[f]);
+                    if ((ret = g2c_open_index(file[f], index_file[f], 0, &g2cid)))
+                        return ret;
                 }
                 else
                 {
                     printf("\ttesting degrib2 on file %s downloaded via FTP without using index...", file[f]);
                     if ((ret = g2c_open(file[f], 0, &g2cid)))
                         return ret;
+		    if ((ret = g2c_write_index(g2cid, 0, index_file[f])))
+                        return ret;
                 }
 
                 /* Check some stuff. */
                 if ((ret = g2c_inq(g2cid, &num_msg)))
                     return ret;
-                printf("num_msg %d\n", num_msg);
+		if (num_msg != expected_nmsg[f])
+		{
+		    printf("num_msg %d expected %d\n", num_msg, expected_nmsg[f]);
+		    return G2C_ERROR;
+		}
 
                 /* Output a degrib2 file. */
-                /* g2c_set_log_level(15); */
                 sprintf(degrib2_file, "%s.degrib2", basename(file[f]));
                 if ((ret = g2c_degrib2(g2cid, degrib2_file)))
                     return ret;
@@ -75,20 +92,24 @@ main()
                 if ((ret = compare_degrib2_files2(degrib2_file, ref_degrib2_file)))
 		{
 		    printf("compare failed!\n");
-		    return ret;
+		    /*return ret;*/
 		}
+		printf("\tok!\n");
             }
-            printf("\tok!\n");
         }
     }
     printf("ok!\n");
-#ifdef LARGE_FTP_TESTS
+#ifdef LARGE_FTP_TESTS1
     printf("Testing degrib2 on very large files downloaded via FTP...\n");
     {
 #define NUM_LARGE_FILES 1
+#define FV3_NUM_MSG 1081
         char file[NUM_LARGE_FILES][MAX_FILENAME_LEN + 1] = {
             "data/fv3lam.t00z.prslev.f000.grib2"
         };
+        char index_file[NUM_LARGE_FILES][MAX_FILENAME_LEN + 1] = {
+            "fv3lam.t00z.prslev.f000.grib2.idx"
+        };
         int g2cid;
         int num_msg;
         int f;
@@ -98,32 +119,36 @@ main()
         /* g2c_set_log_level(10); */
         for (f = 0; f < NUM_LARGE_FILES; f++)
         {
-            /* for (t = 0; t < 2; t++) */
-            for (t = 0; t < 1; t++)
+            for (t = 0; t < 2; t++)
             {
                 char degrib2_file[MAX_FILENAME_LEN + 9];
                 char ref_degrib2_file[MAX_FILENAME_LEN + 20];
 
                 /* Open the data file with and without the index file. */
                 if (t)
                 {
+		    /* Open the data with the index file. */
                     printf("\ttesting degrib2 on file %s downloaded via FTP using index...", file[f]);
-                    /* if ((ret = g2c_open_index(file[f], REF_GDAS_INDEX_FILE, 0, &g2cid))) */
-                    /*     return ret; */
+                    if ((ret = g2c_open_index(file[f], index_file[f], 0, &g2cid)))
+                        return ret;
                 }
                 else
                 {
+		    /* Open the data and create the index file. */
                     printf("\ttesting degrib2 on file %s downloaded via FTP without using index...", file[f]);
                     if ((ret = g2c_open(file[f], 0, &g2cid)))
                         return ret;
-                }
+
+		    if ((ret = g2c_write_index(g2cid, 0, index_file[f])))
+                        return ret;
+		}
 
                 /* Check some stuff. */
                 if ((ret = g2c_inq(g2cid, &num_msg)))
                     return ret;
                 printf("num_msg %d\n", num_msg);
-                /* if (num_msg != GDAS_NUM_MSG) */
-                /*     return G2C_ERROR; */
+                if (num_msg != FV3_NUM_MSG)
+                    return G2C_ERROR;
 
                 /* Output a degrib2 file. */
                 sprintf(degrib2_file, "%s.degrib2", basename(file[f]));