From 88dacafa9fc404c4c53ddd13abc5740a4d9d97c3 Mon Sep 17 00:00:00 2001 From: Cole Harvey Date: Thu, 8 Feb 2024 13:24:56 -0500 Subject: [PATCH 1/5] added fix for blocking factor of 0 --- postprocessing/mppnccombine/mppnccombine.c | 43 +++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/postprocessing/mppnccombine/mppnccombine.c b/postprocessing/mppnccombine/mppnccombine.c index 7c8cea12..93eaf28a 100644 --- a/postprocessing/mppnccombine/mppnccombine.c +++ b/postprocessing/mppnccombine/mppnccombine.c @@ -1004,29 +1004,28 @@ int process_vars(struct fileinfo *ncinfile, struct fileinfo *ncoutfile, recdimsize=ncinfile->dimsize[ncinfile->recdim]; /* Check the number of records */ - if (*nrecs==1) - { - *nrecs=recdimsize; - - if ((*bf) >= 1) - { - if ((*bf) > (*nrecs)) { - fprintf(stderr, "blocking factor (k) > total records (%d). Setting blocking factor to %d.\n", - *nrecs, *nrecs); - *bf = *nrecs; - } - if (((*nrecs) % (*bf)) != 0) *nblocks = (int)((*nrecs)/(*bf)) + 1; - else *nblocks = (int)((*nrecs)/(*bf)); - } - else - { - /* bf was set to zero, so we do full buffering */ - *bf = min(MAX_BF,*nrecs); // we use the maximum blocking factor in our capacity - /* normally we'll have one block, unless we hit MAX_BF */ - *nblocks = (int)((*nrecs)/(*bf)); - } - if (verbose) fprintf(stderr, "blocking factor=%d, num. blocks=%d, num. records=%d\n",*bf,*nblocks, *nrecs); + if (*nrecs==1) { + *nrecs=recdimsize; + + /* adjust bf */ + if ((*bf) >= 1) { + if ((*bf) > (*nrecs)) { + fprintf(stderr, "blocking factor (k) > total records (%d). Setting blocking factor to %d.\n", + *nrecs, *nrecs); + *bf = *nrecs; + } } + else { + /* bf was set to zero, so we do full buffering */ + *bf = min(MAX_BF,*nrecs); // we use the maximum blocking factor in our capacity + /* normally we'll have one block, unless we hit MAX_BF */ + } + /* find nblocks */ + if (((*nrecs) % (*bf)) != 0) *nblocks = (int)((*nrecs)/(*bf)) + 1; + else *nblocks = (int)((*nrecs)/(*bf)); + + if (verbose) fprintf(stderr, "blocking factor=%d, num. blocks=%d, num. records=%d\n",*bf,*nblocks, *nrecs); + } else if (recdimsize != *nrecs) { From 284acae47aa39e746f489c40e59108fdf5e916a4 Mon Sep 17 00:00:00 2001 From: Cole Harvey Date: Thu, 8 Feb 2024 16:23:36 -0500 Subject: [PATCH 2/5] created test for mppnccombine edge case --- t/Makefile.am | 3 +- t/Test34-mppnccombine_blocking_factor.sh | 36 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100755 t/Test34-mppnccombine_blocking_factor.sh diff --git a/t/Makefile.am b/t/Makefile.am index 7d88c42f..39907621 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -68,7 +68,8 @@ TESTS = Test01-check_programs_exist.sh \ Test29-make_vgrid.sh \ Test31-fregrid_stretched.sh \ Test32-fregrid_no_stretched.sh \ - Test33-reference_make_hgrid.sh + Test33-reference_make_hgrid.sh \ + Test34-mppnccombine_blocking_factor.sh EXTRA_DIST = $(TESTS) \ Test02-input\ diff --git a/t/Test34-mppnccombine_blocking_factor.sh b/t/Test34-mppnccombine_blocking_factor.sh new file mode 100755 index 00000000..ee2ffc76 --- /dev/null +++ b/t/Test34-mppnccombine_blocking_factor.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bats + +#*********************************************************************** +# GNU Lesser General Public License +# +# This file is part of the GFDL FRE NetCDF tools package (FRE-NCTools). +# +# FRE-NCTools is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# FRE-NCTools is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with FRE-NCTools. If not, see +# . +#*********************************************************************** +# load default setup/teardown and create numbered input from ncls with given filename +load test_utils + +@test "mppnccombine combines" { + + generate_all_from_ncl_num mppnccombine Test02-input + + #Combine the files into 1 + mppnccombine \ + -k 0 \ + mppnccombine_output.nc \ + mppnccombine.nc.???? + run_and_check [ -e mppnccombine_output.nc ] + ncdump -h mppnccombine_output.nc +} From 7dde814b6e8314f7d3b52c1e230dbd16f0ac9546 Mon Sep 17 00:00:00 2001 From: Cole Harvey Date: Fri, 9 Feb 2024 10:52:03 -0500 Subject: [PATCH 3/5] created a subtest in Test02 instead of separate unit test --- t/Makefile.am | 3 +- t/Test02-mppnccombine.sh | 13 +++++++++ t/Test34-mppnccombine_blocking_factor.sh | 36 ------------------------ 3 files changed, 14 insertions(+), 38 deletions(-) delete mode 100755 t/Test34-mppnccombine_blocking_factor.sh diff --git a/t/Makefile.am b/t/Makefile.am index 39907621..7d88c42f 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -68,8 +68,7 @@ TESTS = Test01-check_programs_exist.sh \ Test29-make_vgrid.sh \ Test31-fregrid_stretched.sh \ Test32-fregrid_no_stretched.sh \ - Test33-reference_make_hgrid.sh \ - Test34-mppnccombine_blocking_factor.sh + Test33-reference_make_hgrid.sh EXTRA_DIST = $(TESTS) \ Test02-input\ diff --git a/t/Test02-mppnccombine.sh b/t/Test02-mppnccombine.sh index 6785e76a..06257d87 100755 --- a/t/Test02-mppnccombine.sh +++ b/t/Test02-mppnccombine.sh @@ -33,3 +33,16 @@ load test_utils run_and_check [ -e mppnccombine_output.nc ] ncdump -h mppnccombine_output.nc } + +@test "mppnccombine combines with blocking factor 0" { + + generate_all_from_ncl_num mppnccombine Test02-input + + #Combine the files into 1 + mppnccombine \ + -k 0 \ + mppnccombine_output.nc \ + mppnccombine.nc.???? + run_and_check [ -e mppnccombine_output.nc ] + ncdump -h mppnccombine_output.nc +} diff --git a/t/Test34-mppnccombine_blocking_factor.sh b/t/Test34-mppnccombine_blocking_factor.sh deleted file mode 100755 index ee2ffc76..00000000 --- a/t/Test34-mppnccombine_blocking_factor.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bats - -#*********************************************************************** -# GNU Lesser General Public License -# -# This file is part of the GFDL FRE NetCDF tools package (FRE-NCTools). -# -# FRE-NCTools is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or (at -# your option) any later version. -# -# FRE-NCTools is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with FRE-NCTools. If not, see -# . -#*********************************************************************** -# load default setup/teardown and create numbered input from ncls with given filename -load test_utils - -@test "mppnccombine combines" { - - generate_all_from_ncl_num mppnccombine Test02-input - - #Combine the files into 1 - mppnccombine \ - -k 0 \ - mppnccombine_output.nc \ - mppnccombine.nc.???? - run_and_check [ -e mppnccombine_output.nc ] - ncdump -h mppnccombine_output.nc -} From bdee4059ba59e05f25d43b66302f10f403d0d179 Mon Sep 17 00:00:00 2001 From: Cole Harvey Date: Mon, 12 Feb 2024 09:07:38 -0500 Subject: [PATCH 4/5] minor syntax fix --- postprocessing/mppnccombine/mppnccombine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postprocessing/mppnccombine/mppnccombine.c b/postprocessing/mppnccombine/mppnccombine.c index 93eaf28a..49634f47 100644 --- a/postprocessing/mppnccombine/mppnccombine.c +++ b/postprocessing/mppnccombine/mppnccombine.c @@ -1010,8 +1010,8 @@ int process_vars(struct fileinfo *ncinfile, struct fileinfo *ncoutfile, /* adjust bf */ if ((*bf) >= 1) { if ((*bf) > (*nrecs)) { - fprintf(stderr, "blocking factor (k) > total records (%d). Setting blocking factor to %d.\n", - *nrecs, *nrecs); + fprintf(stderr, "blocking factor (k) > total records (%d). Setting blocking factor to %d.\n", + *nrecs, *nrecs); *bf = *nrecs; } } From 5fc772adb3dd056560148b304706ebbce6fbe5e8 Mon Sep 17 00:00:00 2001 From: Cole Harvey Date: Mon, 12 Feb 2024 09:12:18 -0500 Subject: [PATCH 5/5] minor syntax fix --- postprocessing/mppnccombine/mppnccombine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postprocessing/mppnccombine/mppnccombine.c b/postprocessing/mppnccombine/mppnccombine.c index 49634f47..d797fbfc 100644 --- a/postprocessing/mppnccombine/mppnccombine.c +++ b/postprocessing/mppnccombine/mppnccombine.c @@ -1010,8 +1010,8 @@ int process_vars(struct fileinfo *ncinfile, struct fileinfo *ncoutfile, /* adjust bf */ if ((*bf) >= 1) { if ((*bf) > (*nrecs)) { - fprintf(stderr, "blocking factor (k) > total records (%d). Setting blocking factor to %d.\n", - *nrecs, *nrecs); + fprintf(stderr, "blocking factor (k) > total records (%d). Setting blocking factor to %d.\n", + *nrecs, *nrecs); *bf = *nrecs; } }