Skip to content

Commit

Permalink
Merge pull request #304 from rem1776/add-nf-syncs
Browse files Browse the repository at this point in the history 
Add sync calls before netcdf files are closed after a write and add error checking
  • Loading branch information
@rem1776
rem1776 authored Jul 18, 2024
2 parents 3055052 + d2bb143 commit 66fba51
Show file tree
Hide file tree
Showing 13 changed files with 150 additions and 118 deletions.
61 changes: 31 additions & 30 deletions postprocessing/land_utils/combine-ncc.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
!
!-----------------------------------------------------------------------

#define __NF_ASRT__(ierr) call nfu_check_err(ierr,__FILE__,__LINE__)
#define CHECK_NF_ERRSTAT(ierr) call nfu_check_err(ierr,__FILE__,__LINE__)
program combine_res

use nfu_mod
Expand Down Expand Up @@ -82,8 +82,8 @@ program combine_res
! otherwise it's 32-bit
allocate(input(nfiles))
do i = 1,nfiles
__NF_ASRT__(nf_open(files(i),NF_NOWRITE,input(i)))
__NF_ASRT__(nf_inq_format(input(i),in_format))
CHECK_NF_ERRSTAT(nf_open(files(i),NF_NOWRITE,input(i)))
CHECK_NF_ERRSTAT(nf_inq_format(input(i),in_format))
enddo

if (in_format==NF_FORMAT_NETCDF4) then
Expand All @@ -104,18 +104,18 @@ program combine_res

! mpp_io supports environment variables to set these. For Riga, we'll simply use the defaults`

__NF_ASRT__(nf__create(outfile,cmode,0,blksz,ncid))
CHECK_NF_ERRSTAT(nf__create(outfile,cmode,0,blksz,ncid))

! Create netcdf structure in the output NetCDF file, using last input file
! as a template.

! clone all dimensions; for compressed dimensions calculate the total length in all
! input files
__NF_ASRT__(nf_inq_ndims(input(nfiles),ndims))
CHECK_NF_ERRSTAT(nf_inq_ndims(input(nfiles),ndims))
allocate(dim(ndims))
do dimid = 1,ndims
associate(d => dim(dimid))
__NF_ASRT__(nfu_inq_dim(input(nfiles),dimid,dimname=d%name))
CHECK_NF_ERRSTAT(nfu_inq_dim(input(nfiles),dimid,dimname=d%name))
call inquire_dimension(input(:), d%name, &
len=dimlen, siz=d%buflen, compressed=d%compressed, is_unlim=has_records)
d%len = dimlen
Expand All @@ -133,32 +133,32 @@ program combine_res
endif
if(verbosity>0)&
write(*,'(x,a,i8)')'defining dimension "'//trim(d%name)//'" with length',d%len
__NF_ASRT__(nf_def_dim(ncid,d%name,dimlen,i)) ! i is just a dummy var for dimid, unused
CHECK_NF_ERRSTAT(nf_def_dim(ncid,d%name,dimlen,i)) ! i is just a dummy var for dimid, unused
end associate
enddo

! clone all variable definitions
__NF_ASRT__(nf_inq_nvars(input(nfiles),nvars))
CHECK_NF_ERRSTAT(nf_inq_nvars(input(nfiles),nvars))
do i = 1,nvars
__NF_ASRT__(nfu_clone_var(input(nfiles),i,ncid))
CHECK_NF_ERRSTAT(nfu_clone_var(input(nfiles),i,ncid))
! NOTE: since cloning of variable definition relies on dimension names,
! each variable tile and compressed dimensions automaticaly get the right
! size, as defined while creating dimensions in the output file
enddo

! clone all global attributes
__NF_ASRT__(nf_inq_natts(input(nfiles),ngatts))
CHECK_NF_ERRSTAT(nf_inq_natts(input(nfiles),ngatts))
do i = 1,ngatts
__NF_ASRT__(nf_inq_attname(input(nfiles),NF_GLOBAL,i,attname))
__NF_ASRT__(nf_copy_att(input(nfiles),NF_GLOBAL,attname,ncid,NF_GLOBAL))
CHECK_NF_ERRSTAT(nf_inq_attname(input(nfiles),NF_GLOBAL,i,attname))
CHECK_NF_ERRSTAT(nf_copy_att(input(nfiles),NF_GLOBAL,attname,ncid,NF_GLOBAL))
enddo

! ---- end of definition stage
__NF_ASRT__(nf__enddef(ncid,HEADERPAD,4,0,4))
CHECK_NF_ERRSTAT(nf__enddef(ncid,HEADERPAD,4,0,4))

! copy all uncompressed vars
do varid = 1, nvars
__NF_ASRT__(nfu_inq_var(input(nfiles),varid,name=varname,ndims=ndims,dimids=dimids,dimlens=dimlens,recsize=recsize,nrec=nrec,xtype=xtype))
CHECK_NF_ERRSTAT(nfu_inq_var(input(nfiles),varid,name=varname,ndims=ndims,dimids=dimids,dimlens=dimlens,recsize=recsize,nrec=nrec,xtype=xtype))
n = 0
do k = 1,ndims
if (dim(dimids(k))%compressed) n = n+1
Expand All @@ -175,15 +175,15 @@ program combine_res
! dimension anyway, and (3) there is no convenient interface (yet) in
! nfu utilities for by-record i/o for CHAR variables.
allocate(text(recsize*nrec))
__NF_ASRT__(nf_get_var_text(input(nfiles),varid,text))
__NF_ASRT__(nfu_inq_var(ncid,varname,id=varid1))
__NF_ASRT__(nf_put_var_text(ncid,varid1,text))
CHECK_NF_ERRSTAT(nf_get_var_text(input(nfiles),varid,text))
CHECK_NF_ERRSTAT(nfu_inq_var(ncid,varname,id=varid1))
CHECK_NF_ERRSTAT(nf_put_var_text(ncid,varid1,text))
deallocate(text)
else
allocate(buffer(recsize))
do rec=1,nrec
__NF_ASRT__(nfu_get_rec_r8(input(nfiles),varid,rec,buffer))
__NF_ASRT__(nfu_put_rec_r8(ncid,varname,rec,buffer))
CHECK_NF_ERRSTAT(nfu_get_rec_r8(input(nfiles),varid,rec,buffer))
CHECK_NF_ERRSTAT(nfu_put_rec_r8(ncid,varname,rec,buffer))
enddo
deallocate(buffer)
endif
Expand Down Expand Up @@ -217,12 +217,12 @@ program combine_res

! process all vars that depend on this compressed dimension
do varid = 1, nvars
__NF_ASRT__(nfu_inq_var(input(nfiles),varid,name=varname,ndims=ndims,dimids=dimids,dimlens=dimlens))
CHECK_NF_ERRSTAT(nfu_inq_var(input(nfiles),varid,name=varname,ndims=ndims,dimids=dimids,dimlens=dimlens))
if (.not.any(dimids(1:ndims)==dimid)) cycle ! skip variables that do not depend on our compressed dim
if(verbosity>0) write(*,'(2x,a)')'copy compressed variable "'//trim(varname)//'"'

! get the output variable ID
__NF_ASRT__(nfu_inq_var(ncid,varname,id=ovarid))
CHECK_NF_ERRSTAT(nfu_inq_var(ncid,varname,id=ovarid))

! find index of the compressed dimension
cdim = 1
Expand All @@ -247,8 +247,8 @@ program combine_res
k = 1
do ifile = 1,nfiles
cnt(cdim) = sizes(ifile)
__NF_ASRT__(nfu_inq_var(input(ifile),varname,id=varid1))
__NF_ASRT__(nf_get_vara_double(input(ifile),varid1,start,cnt,buffer(k)))
CHECK_NF_ERRSTAT(nfu_inq_var(input(ifile),varname,id=varid1))
CHECK_NF_ERRSTAT(nf_get_vara_double(input(ifile),varid1,start,cnt,buffer(k)))
k = k+sizes(ifile)
enddo
! reshuffle variable values in desired order
Expand All @@ -257,14 +257,15 @@ program combine_res
enddo
! write slice to output file
cnt(cdim) = d%len
__NF_ASRT__(nf_put_vara_double(ncid,ovarid,start,cnt,obuffer))
CHECK_NF_ERRSTAT(nf_put_vara_double(ncid,ovarid,start,cnt,obuffer))
enddo
enddo
deallocate(rank, buffer, obuffer)
end associate
enddo

i = nf_close(ncid)
CHECK_NF_ERRSTAT(nf_sync(ncid))
CHECK_NF_ERRSTAT(nf_close(ncid))
contains ! -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

subroutine inquire_dimension(input,name,len,compressed,siz,sizes,is_unlim)
Expand All @@ -288,7 +289,7 @@ subroutine inquire_dimension(input,name,len,compressed,siz,sizes,is_unlim)
integer :: i

nfiles = size(input)
__NF_ASRT__(nfu_inq_dim(input(nfiles), name, dimlen=len_, is_unlim=is_unlim))
CHECK_NF_ERRSTAT(nfu_inq_dim(input(nfiles), name, dimlen=len_, is_unlim=is_unlim))
if (present(len)) len = len_
if (present(siz)) siz = len_
if (present(sizes)) sizes(:)=len_
Expand All @@ -301,15 +302,15 @@ subroutine inquire_dimension(input,name,len,compressed,siz,sizes,is_unlim)

! calculate total size of the compressed dimension across all files
do i=1,nfiles
__NF_ASRT__(nfu_inq_dim(input(i),name,dimlen=sizes_(i)))
CHECK_NF_ERRSTAT(nfu_inq_dim(input(i),name,dimlen=sizes_(i)))
enddo
if (present(sizes)) sizes(:) = sizes_(:)
if (present(siz)) siz = sum(sizes_(:))
if (present(len)) then
len = 0
allocate(buff(maxval(sizes_(:))))
do i=1,nfiles
__NF_ASRT__(nfu_get_var_int(input(i),name,buff))
CHECK_NF_ERRSTAT(nfu_get_var_int(input(i),name,buff))
len = len + count(buff(1:sizes_(i))>=0)
enddo
deallocate(buff)
Expand All @@ -328,8 +329,8 @@ subroutine reorder_compressed_index(input,name,rank)
allocate(buff(size(rank)))
k = 1
do i=1,nfiles
__NF_ASRT__(nfu_get_var_int(input(i),name,buff(k:)))
__NF_ASRT__(nfu_inq_dim(input(i),name,dimlen=dimlen))
CHECK_NF_ERRSTAT(nfu_get_var_int(input(i),name,buff(k:)))
CHECK_NF_ERRSTAT(nfu_inq_dim(input(i),name,dimlen=dimlen))
k = k+dimlen
enddo

Expand Down
57 changes: 29 additions & 28 deletions postprocessing/land_utils/decompress-ncc.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
!
!-----------------------------------------------------------------------

#define __NF_ASRT__(ierr) call nfu_check_err(ierr,__FILE__,__LINE__)
#define CHECK_NF_ERRSTAT(ierr) call nfu_check_err(ierr,__FILE__,__LINE__)
program decompress

use nfu_mod
Expand Down Expand Up @@ -67,8 +67,8 @@ program decompress
! open all input files
allocate(input(nfiles))
do i = 1,nfiles
__NF_ASRT__(nf_open(files(i),NF_NOWRITE,input(i)))
__NF_ASRT__(nf_inq_format(input(i),in_format))
CHECK_NF_ERRSTAT(nf_open(files(i),NF_NOWRITE,input(i)))
CHECK_NF_ERRSTAT(nf_inq_format(input(i),in_format))
enddo

if (in_format==NF_FORMAT_NETCDF4) then
Expand All @@ -86,41 +86,41 @@ program decompress
endif

! create output file
__NF_ASRT__(nf__create(outfile,cmode,0,blksz,ncid))
CHECK_NF_ERRSTAT(nf__create(outfile,cmode,0,blksz,ncid))

! Create netcdf structure in the output NetCDF file, using last input file
! as a template.

! clone all dimensions except compressed ones; compressed are just skipped
__NF_ASRT__(nf_inq_ndims(input(nfiles),ndims))
CHECK_NF_ERRSTAT(nf_inq_ndims(input(nfiles),ndims))
do dimid = 1,ndims
__NF_ASRT__(nfu_inq_dim(input(nfiles),dimid,dimname=dimname,dimlen=dimlen,is_unlim=has_records))
CHECK_NF_ERRSTAT(nfu_inq_dim(input(nfiles),dimid,dimname=dimname,dimlen=dimlen,is_unlim=has_records))
if(nfu_inq_att(input(nfiles),dimname,'compress')==NF_NOERR) cycle
if(has_records)&
dimlen=NF_UNLIMITED
if(debug>0)&
write(*,*)'defining dimension "'//trim(dimname)//'" with length',dimlen
__NF_ASRT__(nf_def_dim(ncid,dimname,dimlen,i)) ! i is just a space for id
CHECK_NF_ERRSTAT(nf_def_dim(ncid,dimname,dimlen,i)) ! i is just a space for id
enddo

! clone all variable definitions, replacing compressed dimensions with sets
! of uncompressed ones
__NF_ASRT__(nf_inq_nvars(input(nfiles),nvars))
CHECK_NF_ERRSTAT(nf_inq_nvars(input(nfiles),nvars))
do varid = 1,nvars
iret = nfu_inq_compressed_var(input(nfiles),varid,varname,xtype,ndims,dimids,&
natts=natts,is_dim=is_dim, is_compressed=is_compressed)
if(debug>0)&
write(*,*)'defining variable "'//trim(varname)//'"'
__NF_ASRT__(iret) ! just because the line is going to be too long
CHECK_NF_ERRSTAT(iret) ! just because the line is going to be too long
if(is_dim.and.is_compressed) cycle
do i = 1,ndims
__NF_ASRT__(nf_inq_dimname(input(nfiles),dimids(i),dimname))
__NF_ASRT__(nf_inq_dimid(ncid,dimname,dimids(i)))
CHECK_NF_ERRSTAT(nf_inq_dimname(input(nfiles),dimids(i),dimname))
CHECK_NF_ERRSTAT(nf_inq_dimid(ncid,dimname,dimids(i)))
enddo
__NF_ASRT__(nf_def_var(ncid,varname,xtype,ndims,dimids,varid1))
CHECK_NF_ERRSTAT(nf_def_var(ncid,varname,xtype,ndims,dimids,varid1))
do i=1,natts
__NF_ASRT__(nf_inq_attname(input(nfiles),varid,i,attname))
__NF_ASRT__(nf_copy_att(input(nfiles),varid,attname,ncid,varid1))
CHECK_NF_ERRSTAT(nf_inq_attname(input(nfiles),varid,i,attname))
CHECK_NF_ERRSTAT(nf_copy_att(input(nfiles),varid,attname,ncid,varid1))
enddo
if(add_missing_value.and.is_compressed) then
! check if missing_value or _FillValue attributes are present
Expand All @@ -136,39 +136,39 @@ program decompress
missing = NF_FILL_INT
end select
! and add it to the output variable
__NF_ASRT__(nf_put_att_double(ncid,varid1,'missing_value',xtype,1,missing))
CHECK_NF_ERRSTAT(nf_put_att_double(ncid,varid1,'missing_value',xtype,1,missing))
endif
endif
enddo

! clone all global attributes
__NF_ASRT__(nf_inq_natts(input(nfiles),natts))
CHECK_NF_ERRSTAT(nf_inq_natts(input(nfiles),natts))
do i = 1,natts
__NF_ASRT__(nf_inq_attname(input(nfiles),NF_GLOBAL,i,attname))
__NF_ASRT__(nf_copy_att(input(nfiles),NF_GLOBAL,attname,ncid,NF_GLOBAL))
CHECK_NF_ERRSTAT(nf_inq_attname(input(nfiles),NF_GLOBAL,i,attname))
CHECK_NF_ERRSTAT(nf_copy_att(input(nfiles),NF_GLOBAL,attname,ncid,NF_GLOBAL))
enddo

! ---- end of definition stage
__NF_ASRT__(nf__enddef(ncid,HEADERPAD,4,0,4))
CHECK_NF_ERRSTAT(nf__enddef(ncid,HEADERPAD,4,0,4))


! grow unlimited dimension, if necessary
do varid=1,nvars
__NF_ASRT__(nfu_inq_compressed_var(input(nfiles),varid,name=varname,dimlens=dimlens,has_records=has_records))
CHECK_NF_ERRSTAT(nfu_inq_compressed_var(input(nfiles),varid,name=varname,dimlens=dimlens,has_records=has_records))
if(has_records) then
! just write an integer at the very end of the variable -- that extends the
! record dimensions as well
__NF_ASRT__(nf_inq_varid(ncid,varname,varid))
__NF_ASRT__(nf_put_var1_int(ncid,varid,dimlens,0))
CHECK_NF_ERRSTAT(nf_inq_varid(ncid,varname,varid))
CHECK_NF_ERRSTAT(nf_put_var1_int(ncid,varid,dimlens,0))
exit ! this loop
endif
enddo

! gather and copy data
__NF_ASRT__(nf_inq_nvars(ncid,nvars))
CHECK_NF_ERRSTAT(nf_inq_nvars(ncid,nvars))
do varid = 1,nvars
! __NF_ASRT__(nfu_inq_compressed_var(ncid,varid,name=varname,varsize=vsize))
__NF_ASRT__(nfu_inq_var(ncid,varid,varname,xtype,varsize=vsize))
! CHECK_NF_ERRSTAT(nfu_inq_compressed_var(ncid,varid,name=varname,varsize=vsize))
CHECK_NF_ERRSTAT(nfu_inq_var(ncid,varid,varname,xtype,varsize=vsize))
if(debug>0) &
write(*,*)'processing var "'//trim(varname)//'"'
allocate(buffer(vsize),mask(vsize))
Expand Down Expand Up @@ -200,14 +200,15 @@ program decompress

! read the variable
do i=1,nfiles
__NF_ASRT__(nfu_get_compressed_var_r8n(input(i),varname,buffer,mask,ocean=do_oceanValue,ocean_value=ocean_value))
CHECK_NF_ERRSTAT(nfu_get_compressed_var_r8n(input(i),varname,buffer,mask,ocean=do_oceanValue,ocean_value=ocean_value))
enddo
! write the variable
__NF_ASRT__(nfu_put_var_r8(ncid,varname,buffer))
CHECK_NF_ERRSTAT(nfu_put_var_r8(ncid,varname,buffer))
deallocate(buffer,mask)
enddo

__NF_ASRT__(nf_close(ncid))
CHECK_NF_ERRSTAT(nf_sync(ncid))
CHECK_NF_ERRSTAT(nf_close(ncid))

contains ! -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

Expand Down
14 changes: 7 additions & 7 deletions postprocessing/land_utils/is-compressed.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
#include <unistd.h>
#include <netcdf.h>

#define __NC_ASRT__(ierr) check_error(ierr,__FILE__,__LINE__)
#define CHECK_NC_ERRSTAT(ierr) check_error(ierr,__FILE__,__LINE__)

void usage(const char* name)
{
Expand All @@ -49,8 +49,8 @@ void usage(const char* name)
void check_error(int ierr, const char* file, int line)
{
if (ierr!=NC_NOERR) {
fprintf(stderr,"ERROR :: %s\n",
nc_strerror(ierr));
fprintf(stderr,"ERROR :: %s\nfile: %s, line: %d\n",
nc_strerror(ierr), file, line);
exit(ierr);
}
}
Expand Down Expand Up @@ -85,18 +85,18 @@ int main(int argc, char* argv[])

int optind = argc - 1;

__NC_ASRT__(nc_open(argv[optind],NC_NOWRITE,&ncid));
__NC_ASRT__(nc_inq_ndims(ncid,&ndims));
CHECK_NC_ERRSTAT(nc_open(argv[optind],NC_NOWRITE,&ncid));
CHECK_NC_ERRSTAT(nc_inq_ndims(ncid,&ndims));
if (verbose) printf("Found %d dimensions\n",ndims);
for (dimid=0; dimid<ndims; dimid++) {
__NC_ASRT__(nc_inq_dimname(ncid,dimid,name));
CHECK_NC_ERRSTAT(nc_inq_dimname(ncid,dimid,name));
if (verbose) printf("Examining \"%s\"\n",name);
if (nc_inq_varid(ncid,name,&varid)==NC_NOERR &&
nc_inq_attid(ncid,varid,"compress",&attid)==NC_NOERR) {
return 0;
}
}
__NC_ASRT__(nc_close(ncid));
CHECK_NC_ERRSTAT(nc_close(ncid));

return -1;
}
Loading

0 comments on commit 66fba51

Please sign in to comment.