makenc.py

"""
Created on 2/19/2016
This script is desinged to create netCDF files using the netCDF4 module from python as
part of the Coastal Model Test Bed (CMTB)

@author: Spicer Bak
@contact: Spicer.Bak@usace.army.mil
"""
import numpy as np
import netCDF4 as nc
import csv, yaml
import datetime as DT
import time as ttime

def readflags(flagfname, header=1):
    """This function reads the flag file from the data in to the STWAVE CMTB runs

    Args:
      flagfname: the relative/absolute location of the flags file
      header:  (Default value = 1)

    Returns:
      flags of data dtype=dictionary

    """
    times, waveflag, windflag, WLflag, curflag,allflags = [], [],[],[],[],[]

    try:
        with open(flagfname, 'rb') as f:
            reader = csv.reader(f)  # opening file
            for row in reader:  # iteratin
                # go over the open file
                if len(row) > 1 and row[0] != 'Date':
                    waveflag.append(int(row[2]))  # appending wave data flag
                    windflag.append(int(row[3]))  # appending Wind data flag
                    WLflag.append(int(row[4]))  # appending Water Level Flag data
                    curflag.append(int(row[5]))  # appending ocean Currents flag data
                    times.append(DT.datetime.strptime(row[0]+row[1], '%Y-%m-%d%H%M'))
                    allflags.append([int(row[2]), int(row[3]), int(row[4]), int(row[5])])
        # creating array of flags
        allflags = np.array(allflags)
    except IOError:
        allflags = None
    # putting data into a dictionary
    flags = {'time': times,
             'windflag': windflag,
             'waveflag': waveflag,
             'WLflag': WLflag,
             'curflag': curflag,
             'allflags': allflags
             }
    return flags

def import_template_file(yaml_location):
    """This function loads a yaml file and returns the attributes in dictionary
    written by: ASA


    Step 1 in netCDF file creation, open global and variable yamls

    Args:
      yaml_location: yaml file location

    Returns:
        dictionary with variables from the yaml file

    """
    # load the template
    f = open(yaml_location)
    # use safe_load instead load
    vars_dict = yaml.safe_load(f)
    f.close()
    return vars_dict

def init_nc_file(nc_filename, attributes):
    """Create the netCDF file and write the Global Attributes
    written by ASA

    will initalize netCDF file and set global attributes, write date created and issued to global meta data

    Args:
      nc_filename: output netCDF file name
      attributes: attributes from global yaml load

    Returns:
        open netCDF file ready for writing data

    """

    ncfile = nc.Dataset(nc_filename, 'w', clobber=True)

    # Write some Global Attributes
    for key, value in attributes.iteritems():

        if value is not None:
            setattr(ncfile, key, value)

    dt_today = ttime.strftime("%Y-%m-%d")
    ncfile.date_created = dt_today
    ncfile.date_issued = dt_today

    return ncfile

def write_data_to_nc(ncfile, template_vars, data_dict, write_vars='_variables'):
    """This function actually writes the variables and the variable attributes to
    the netCDF file

    in the yaml, the "[variable]:" needs to be in the data dictionary,
     the output netcdf variable will take the name "name:"
    
     Edited by Spicer Bak

    Args:
      ncfile: this is an alreayd opened netCDF file with already defined dimensions
      template_vars (dict): variable and meta data associated with data_dict
      data_dict (dict): this is a dictionary with keys associated to those hopefully in template_vars, this holds the data
      write_vars: Unknown (Default value = '_variables')

    Returns:
      netCDF file (still open)
      also returns error strings and count that were created during the data writing process

    """
    # Keep track of any errors found
    num_errors = 0
    error_str = ''

    # write some more global attributes if present
    if '_attributes' in template_vars:
        for var in template_vars['_attributes']:
            if var in data_dict:
                setattr(ncfile, var, data_dict[var])

    # List all possible variable attributes in the template
    possible_var_attr = ['standard_name', 'long_name', 'coordinates', 'flag_values', 'flag_meanings', 'description',
                         'notes', 'positive', 'valid_min', 'valid_max', 'calendar', 'description', 'cf_role',
                         'missing_value']

    # Write variables to file
    accept_vars = template_vars['_variables']

    for var in accept_vars:  # only write varibles that were loaded from .yaml file
        if var in data_dict:
            try:
                if "fill_value" in template_vars[var] and "least_significant_digit" in template_vars[var]:
                    new_var = ncfile.createVariable(template_vars[var]["name"],
                                                    template_vars[var]["data_type"],
                                                    template_vars[var]["dim"],
                                                    fill_value=template_vars[var]["fill_value"],
                                                    least_significant_digit=template_vars[var]['least_significant_digit'] )
                elif "fill_value" in template_vars[var]:
                    new_var = ncfile.createVariable(template_vars[var]["name"], template_vars[var]["data_type"],
                            template_vars[var]["dim"], fill_value=template_vars[var]["fill_value"])
                else:
                    new_var = ncfile.createVariable(template_vars[var]["name"],
                                                    template_vars[var]["data_type"],
                                                    template_vars[var]["dim"])

                new_var.units = template_vars[var]["units"]

                # Write the attributes
                for attr in possible_var_attr:  # only write attributes listed in this list above
                    if attr in template_vars[var]:
                        if template_vars[var][attr] == 'NaN':
                            setattr(new_var, attr, np.nan)
                        else:
                            setattr(new_var, attr, template_vars[var][attr])
                # Write the short_name attribute as the variable name
                if 'short_name' in template_vars[var]:
                    new_var.short_name = template_vars[var]["short_name"]
                else:
                    new_var.short_name = template_vars[var]["name"]
                # _____________________________________________________________________________________
                # Write the data (1D, 2D, or 3D)
                #______________________________________________________________________________________
                if var == "station_name":
                    station_id = data_dict[var]
                    data = np.empty((1,), 'S'+repr(len(station_id)))
                    data[0] = station_id
                    new_var[:] = nc.stringtochar(data)
                elif len(template_vars[var]["dim"]) == 0:
                    try:
                        new_var[:] = data_dict[var]
                    except Exception, e:
                        new_var = data_dict[var]

                elif len(template_vars[var]["dim"]) == 1:
                    # catch some possible errors for frequency and direction arrays
                    if template_vars[var]["data_type"] == 'str':
                        for i, c in enumerate(template_vars[var]["data_type"]):
                            new_var[i] = data_dict[var][i]
                    else:
                        try:
                            new_var[:] = data_dict[var]
                        except IndexError:
                            try:
                                new_var[:] = data_dict[var][0][0]
                            except Exception, e:
                                raise e

                elif len(template_vars[var]["dim"]) == 2:
                    # create an empty 2d data set of the correct sizes
                    try:
                        # handles row vs col data, rather than transposing the array just figure out which it is
                        length = data_dict[var][0].shape[1]
                        if data_dict[var][0].shape[0] > length:
                            length = data_dict[var][0].shape[0]

                        x = np.empty([data_dict[var].shape[0], length], dtype=np.float64)
                        for i in range(data_dict[var].shape[0]):
                            # squeeze the 3d array in to 2d as dimension is not needed
                            x[i] = np.squeeze(data_dict[var][i])
                        new_var[:, :] = x
                    except Exception, e:
                        # if the tuple fails must be right...right?
                        new_var[:] = data_dict[var]

                elif len(template_vars[var]["dim"]) == 3:
                    # create an empty 3d data set of the correct sizes
                    # this portion was modified by Spicer Bak
                    assert data_dict[var].shape == new_var.shape, 'The data must have the Same Dimensions  (missing time?)'
                    x = np.empty([data_dict[var].shape[0], data_dict[var].shape[1], data_dict[var].shape[2]], np.float64)
                    for i in range(data_dict[var].shape[0]):
                        x[i] = data_dict[var][i]
                    new_var[:, :, :] = x[:, :, :]

            except Exception, e:
                num_errors += 1
                print('ERROR WRITING VARIABLE: {} - {} \n'.format(var, str(e)))


    return num_errors, error_str

def makenc_field(data_lib, globalyaml_fname, flagfname, ofname, var_yaml_fname):
    """This is a function that takes wave nest dictionary and Tp_nest dictionnary and creates the high resolution
    near shore field data from the Coastal Model Test Bed

    Args:
      data_lib: data lib is a library of data with keys the same name as associated variables to be written in the
            netCDF file to be created, This function will look for:

            'time', 'DX', 'DY', 'NI', 'NJ', 'bathymetry', 'bathymetryDate', 'waveHs', 'station_name'

      globalyaml_fname: global meta data yaml file name
      ofname: the file name to be created
      flagfname: flag input file to flag data
      var_yaml_fname:  variable meta data yaml file name


    Returns:
        written netCDF file
    """

    # import global atts
    globalatts = import_template_file(globalyaml_fname)
    # import variable data and meta
    var_atts = import_template_file(var_yaml_fname)
    # import flag data
    flags = readflags(flagfname)['allflags']
    data_lib['flags'] = flags
    # figure out my grid spacing and write it to the file
    if np.mean(data_lib['DX']) != np.median(data_lib['DX']):  # variable grid spacing
        globalatts['grid_dx'] = 'variable'
        globalatts['grid_dy'] = 'variable'
    else:
        globalatts['grid_dx'] = data_lib['DX']
        globalatts['grid_dy'] = data_lib['DY']
    globalatts['n_cell_y'] = data_lib['NJ']
    globalatts['n_cell_x'] = data_lib['NI']

    # making bathymetry the length of time so it can be concatnated
    if data_lib['waveHs'].shape[1] != data_lib['bathymetry'].shape[1]:
        data_lib['waveHs']=data_lib['waveHs'][:,:data_lib['bathymetry'].shape[1],:]
    data_lib['bathymetry'] = np.full_like(data_lib['waveHs'], data_lib['bathymetry'], dtype=np.float32 )
    if 'bathymetryDate' in data_lib:
        data_lib['bathymetryDate'] = np.full_like(data_lib['time'], data_lib['bathymetryDate'], dtype=np.float32 )


    #data_lib['bathymetry'] =
    fid = init_nc_file(ofname, globalatts)  # initialize and write inital globals

    #### create dimensions
    tdim = fid.createDimension('time', np.shape(data_lib['waveHs'])[0])
    xdim = fid.createDimension('X_shore', data_lib['NI'])
    ydim = fid.createDimension('Y_shore', data_lib['NJ'])
    inputtypes = fid.createDimension('in_type', np.shape(flags)[1]) # there are 4 input data types for flags
    statnamelen = fid.createDimension('station_name_length', len(data_lib['station_name']))
    #if 'bathymetryDate' in data_lib:
    #    bathyDate_length = fid.createDimension('bathyDate_length', np.shape(data_lib['bathymetry'])[0])

    # bathydate = fid.createDimension('bathyDate_length', np.size(data_lib['bathymetryDate']))

    # write data to the nc file
    write_data_to_nc(fid, var_atts, data_lib)
    # close file
    fid.close()

def makenc_FRFTransect(bathyDict, ofname, globalYaml, varYaml):
    """This function makes netCDF files from csv Transect data library created with testbedUtils.load_FRF_transect

    Args:
      bathyDict: data input matching var yaml, must have 'time' in it for dimension
      ofname: the file name to be created
      globalYaml: global meta data yaml file name
      varYaml: variable meta data yaml file name

    Returns:
        closed netCDF file

    """
    globalAtts = import_template_file(globalYaml)  # loading global meta data attributes from  yaml
    varAtts = import_template_file(varYaml)  # loading variables to write and associated meta data

    # initializing output cshore_ncfile
    fid =init_nc_file(ofname, globalAtts)

    # creating dimensions of data
    tdim = fid.createDimension('time', np.shape(bathyDict['time'])[0])

    # write data to the cshore_ncfile
    write_data_to_nc(fid, varAtts, bathyDict)
    # close file
    fid.close()

def makenc_FRFGrid(gridDict, ofname, globalYaml, varYaml):
    """This is a function that makes netCDF files from the FRF Natural neighbor tool created by
    Spicer Bak using the pyngl library. the transect dictionary is created using the natural
    neighbor tool in FRF_natneighbor.py

    Args:
      gridDict: data dictionary matching varYaml requires
        'zgrid', 'ygrid', 'xgrid', 'StateplaneE', 'StateplaneN', 'Lat', 'Lon', 'FRF_X', 'FRF_Y'
      globalYaml: global meta data yaml file name
      ofname: the file name to be created
      varYaml: variable meta data yaml file name

    Returns:
      netCDF file with gridded data in it

    """
    from testbedutils import geoprocess as gp  # this might be creating a circular import
    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)

    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    # creating dimensions of data
    xShore = fid.createDimension('xShore', np.shape(gridDict['zgrid'])[0])
    yShore = fid.createDimension('yShore', np.shape(gridDict['zgrid'])[1])
    time = fid.createDimension('time', np.size(gridDict['time']))

    # creating lat/lon and state plane coords
    #xgrid, ygrid = np.meshgrid(gridDict['xgrid'], gridDict['ygrid'])
    xx, yy = np.meshgrid(gridDict['xgrid'], gridDict['ygrid'])
    latGrid = np.zeros(np.shape(yy))
    lonGrid = np.zeros(np.shape(xx))
    statePlN = np.zeros(np.shape(yy))
    statePlE = np.zeros(np.shape(xx))
    for iy in range(0, np.size(gridDict['zgrid'], axis=1)):
        for ix in range(0, np.size(gridDict['zgrid'], axis=0)):
            coords = gp.FRFcoord(xx[iy, ix], yy[iy, ix])#, grid[iy, ix]))
            statePlE[iy, ix] = coords['StateplaneE']
            statePlN[iy, ix] = coords['StateplaneN']
            latGrid[iy, ix] = coords['Lat']
            lonGrid[iy, ix] = coords['Lon']
            assert xx[iy, ix] == coords['FRF_X']
            assert yy[iy, ix] == coords['FRF_Y']

    # put these data into the dictionary that matches the yaml
    gridDict['Latitude'] = latGrid[:, 0]
    gridDict['Longitude'] = lonGrid[0, :]
    gridDict['Easting'] = statePlE[:, 0]
    gridDict['Northing'] = statePlN[0, :]
    gridDict['FRF_Xshore'] = gridDict.pop('xgrid')
    gridDict['FRF_Yshore'] = gridDict.pop('ygrid')
    # addding 3rd dimension for time
    a=gridDict.pop('zgrid').T
    gridDict['Elevation'] = np.full([1, a.shape[0], a.shape[1]], fill_value=[a], dtype=np.float32)
    # write data to file
    write_data_to_nc(fid, varAtts, gridDict)
    # close file
    fid.close()

def makenc_Station(stat_data, globalyaml_fname, flagfname, ofname, stat_yaml_fname):
    """This function will make netCDF files from the station output data from the
    Coastal Model Test Bed of STWAVE for the STATion files

    Args:
      stat_data: data lib is a library of data with keys the same name as associated variables to be written in the
            netCDF file to be created, This function will look for:

            'time', 'DX', 'DY', 'NI', 'NJ', 'station_name', 'Northing', 'Easting', 'Longitude', 'Latitude', 'waveDirectionBins', 'waveFrequency'
      flagfname: name/path of flag file
      globalyaml_fname: global yaml name
      stat_yaml_fname: varable yamle name
      ofname: output file name

    Returns:
      a nc file with station data in it

    """
     # import global yaml data
    globalatts = import_template_file(globalyaml_fname)
    # import variable data and meta
    stat_var_atts = import_template_file(stat_yaml_fname)
    # import flag data
    flags = readflags(flagfname)['allflags']
    stat_data['flags'] = flags # this is a library of flags
    globalatts['grid_dx'] = stat_data['DX']
    globalatts['grid_dy'] = stat_data['DY']
    globalatts['n_cell_y'] = stat_data['NJ']
    globalatts['n_cell_x'] = stat_data['NI']
    fid = init_nc_file(ofname, globalatts)  # initialize and write inital globals

    #### create dimensions
    tdim = fid.createDimension('time', np.shape(stat_data['time'])[0])  # None = size of the dimension, what does this gain me if i know it
    inputtypes = fid.createDimension('input_types_length', np.shape(flags)[1]) # there are 4 input dtaa types for flags
    statnamelen = fid.createDimension('station_name_length', len(stat_data['station_name']))
    northing = fid.createDimension('Northing', 1L)
    easting = fid.createDimension('Easting', 1L )
    Lon = fid.createDimension('Longitude', np.size(stat_data['Longitude']))
    Lat = fid.createDimension('Latitude', np.size(stat_data['Latitude']))
    dirbin = fid.createDimension('waveDirectionBins', np.size(stat_data['waveDirectionBins']))
    frqbin = fid.createDimension('waveFrequency', np.size(stat_data['waveFrequency']))
    
    #
    # convert to Lat/lon here

    # write data to the nc file
    write_data_to_nc(fid, stat_var_atts, stat_data)
    # close file
    fid.close()

def convert_FRFgrid(gridFname, ofname, globalYaml, varYaml, plotFlag=False):
    """This function will convert the FRF gridded text product into a NetCDF file

    Args:
      gridFname: input FRF gridded product
      ofname: output netcdf filename
      globalYaml: a yaml file containing global meta data
      varYaml: a yaml file containing variable meta data
      plotFlag: true or false for creation of QA plots (Default value = False)

    Returns:
      None

    """
    # Defining rigid parameters
    raise NotImplementedError('is this depricated?')
    # defining the bounds of the FRF gridded product
    gridYmax = 1100  # maximum FRF Y distance for netCDF file
    gridYmin = -100  # minimum FRF Y distance for netCDF file
    gridXmax = 950  # maximum FRF X distance for netCDF file
    gridXmin = 50  # minimum FRF xdistance for netCDF file
    fill_value= '-999.0'
    # main body
    # load Grid from file
    tempClass = PrepData.inputOutput.genericIO()
    xyz = tempClass.importXYZ(gridFname)

    # make dictionary in right form
    dx = np.median(np.diff(xyz['x']))
    dy = np.max(np.diff(xyz['y']))
    xgrid = np.unique(xyz['x'])
    ygrid = np.unique(xyz['y'])

    # putting the loaded grid into a 2D array
    zgrid = np.zeros((len(xgrid), len(ygrid)))
    rc = 0
    for i in range(np.size(ygrid, axis=0 )):
        for j in range(np.size(xgrid, axis=0)):
            zgrid[j, i] = xyz['z'][rc]
            rc += 1
    if plotFlag == True:
        from matplotlib import pyplot as plt
        plt.pcolor(xgrid, ygrid, zgrid.T)
        plt.colorbar()
        plt.title('FRF GRID %s' % ofname[:-3].split('/')[-1])
        plt.savefig(ofname[:-4] + '_RawGridTxt.png')
        plt.close()
    # aking labels in FRF coords for
    ncXcoord = np.linspace(gridXmin, gridXmax, num=(gridXmax - gridXmin) / dx + 1, endpoint=True)
    ncYcoord = np.linspace(gridYmin, gridYmax, num=(gridYmax - gridYmin) / dy + 1, endpoint=True)
    frame = np.full((np.shape(ncXcoord)[0], np.shape(ncYcoord)[0]), fill_value=fill_value)

    # find the overlap locations between grids
    xOverlap = np.intersect1d(xgrid, ncXcoord)
    yOverlap = np.intersect1d(ygrid, ncYcoord)
    assert len(yOverlap) >= 3, 'The overlap between grid nodes and netCDF grid nodes is short'
    lastX = np.argwhere(ncXcoord == xOverlap[-1])[0][0]
    firstX = np.argwhere(ncXcoord == xOverlap[0])[0][0]
    lastY = np.argwhere(ncYcoord == yOverlap[-1])[0][0]
    firstY = np.argwhere(ncYcoord == yOverlap[0])[0][0]

    # fill the frame grid with the loaded data
    frame[firstX:lastX+1, firstY:lastY+1] = zgrid

    # run data check
    assert set(xOverlap).issubset(ncXcoord), 'The FRF X values in your function do not fit into the netCDF format, please rectify'
    assert set(yOverlap).issubset(ncYcoord), 'The FRF Y values in your function do not fit into the netCDF format, please rectify'

    # putting the data into a dictioary to make a netCDF file
    fields = gridFname.split('_')
    for fld in fields:
        if len(fld) == 8:
            dte = fld  # finding the date in the file name
            break
    gridDict = {'zgrid': frame,
                'xgrid': ncXcoord,
                'ygrid': ncYcoord,
                'time': nc.date2num(DT.datetime(int(dte[:4]), int(dte[4:6]),
                                                       int(dte[6:])), 'seconds since 1970-01-01')}

    # making the netCDF file from the gridded data
    makenc_FRFGrid(gridDict, ofname, globalYaml, varYaml)

def makeDirectionalWavesWHOI(ofname, dataDict, globalYaml, varYaml):
    """
    Args:
      ofname: output file name
      dataDict: input data dictionary matching variable names in yaml
      globalYaml: global yaml for meta data ahead of file
      varYaml: variable data structured the same to that of the dataDict

    Returns:
      None  - writes out the netCDF file

    """
    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)
    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    #### create dimensions
    tdim = fid.createDimension('time', np.shape(dataDict['time'])[0])  # None = size of the dimension, what does this gain me if i know it
    statnamelen = fid.createDimension('station_name_length', len(dataDict['station_name']))
    dirbin = fid.createDimension('waveDirectionBins', np.size(dataDict['directionBands']))
    frqbin = fid.createDimension('waveFrequency', np.size(dataDict['freqBands']))

    # write data to the nc file
    write_data_to_nc(fid, varAtts, dataDict)
    # close file
    fid.close()

def makenc_todaysBathyCMTB(gridDict, ofname, globalYaml, varYaml):
    """Generate bathymetry file for CMTB
    Args:
      gridDict: data dictionary matching varYaml
      ofname: file output name
      globalYaml: yaml containing CF compliant meta data
      varYaml: yaml containing matching data structure to gridDict and CF compliant meta data

    Returns:

    """
    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)

    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    # creating dimensions of data
    xFRF = fid.createDimension('xFRF', gridDict['xFRF'].shape[0])
    yFRF = fid.createDimension('yFRF', gridDict['yFRF'].shape[0])
    time = fid.createDimension('time', np.size(gridDict['time']))
    # write data to file
    write_data_to_nc(fid, varAtts, gridDict)
    # close file
    fid.close()

def makenc_CSHORErun(ofname, dataDict, globalYaml, varYaml):
    """This is a function that makes netCDF files from CSHORE model runs created by
       David Young using all the stuff Spicer Bak used. You have to build dataDict from the different dictionaries
       output by cshore_io.load_CSHORE_results().  YOU DONT HAVE TO HAND IT LAT LON THOUGH!!!

    Args:
      dataDict: keys:
        time: - time steps of the simulation nc file

        xFRF: - xFRF positions of the simulation

        aveE: - depth averaged eastward current!

        stdE: - standard deviation of eastward current

        aveN: - same as above but northward current

        stdN: - same as above but northward

        waveHs: - significant wave heights

        waveMeanDirection:  mean direction of the waves at each cross-shore position

        waterLevel: mean water level at each cross-shore position

        stdWaterLevel: standard deviation of the water surface elevation at each cross-shore position

        setup: wave setup at each cross-shore position

        runup2perc: 2 percent exceedance runup elevation for each model time-step

        runupMean: mean runup elevation for each model time-step

        qbx: cross-shore bed load sediment transport rate

        qsx: cross-shore suspended sediment transport rate

        qby: alongshore bed load sediment transport rate

        qsy: alongshore suspended sediment transport rate

        probabilitySuspension: probability that sediment will be suspended at particular node

        probabilityMovement: probability that sediment will move

        suspendedSedVolume: suspended sediment volume at each cross-shore position

        bottomElevation: the bottom elevation at each xFRF position in the simulation

        surveyNumber:  this is the surveyNumber that the integrated bathymetry for this simulation was built on

        profileNumber: this is either the profileNumber of the survey or the alongshore position of the integratred bathymetry transect that is used as the bed elevation boundary condition

        bathymetryDate: this is the day that the aforementioned survey was taken

        yFRF: this is the yFRF position of the transect itself.  if it is the integrated bathymetry, then this will be identical to the profileNumber

      ofname (str): this is the FULL PATH INCLUDING FILENAME AND EXTENSION to the position where the ncFile will be saved when output
      globalYaml (str): full path to the globalYaml used to build this ncFile
      varYaml (str): full path to the variableYaml used to build this ncFile

    Returns:
      netCDF file with CSHORE model results in it

    """
    from testbedutils import geoprocess as gp  # this might create a circular import
    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)

    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    # note: you have to hand this the yFRF coordinates of the BC gage if you want to get lat/lon..
    lx = np.size(dataDict['xFRF'], axis=0)
    lat = np.zeros(lx)
    lon = np.zeros(lx)
    for ii in range(0, lx):
        coords = gp.FRFcoord(dataDict['xFRF'][ii], dataDict['yFRF'])
        lat[ii] = coords['Lat']
        lon[ii] = coords['Lon']
    dataDict['latitude'] = lat
    dataDict['longitude'] = lon

    # ok, we are HARD CODING the dimensions to ALWAYS be at the 8m ARRAY (xFRF = 914.44 rounded DOWN to 914)
    # we will just fill in the missing values with nans as required
    array8m_loc = 914

    # creating dimensions of data
    new_s = np.shape(range(-50, array8m_loc+1))[0]
    new_t = np.shape(dataDict['waveHs'])[0]
    xFRF = fid.createDimension('xFRF', new_s)
    time = fid.createDimension('time', new_t)

    # check to see if the grid I am importing is smaller than my netCDF grid
    if np.shape(range(-50, array8m_loc+1))[0] == np.shape(dataDict['xFRF']):
        # the model grid is the same as the netCDF grid, so do nothing
        dataDict_n = dataDict
        pass
    else:
        dataDict_n = {'xFRF': np.flipud(np.array(range(-50, array8m_loc+1)) + 0.0),
                      'time': dataDict['time'],
                      'aveE': np.full((new_t, new_s), fill_value=np.nan),
                      'stdE': np.full((new_t, new_s), fill_value=np.nan),
                      'aveN': np.full((new_t, new_s), fill_value=np.nan),
                      'stdN': np.full((new_t, new_s), fill_value=np.nan),
                      'waveHs': np.full((new_t, new_s), fill_value=np.nan),
                      'waveMeanDirection': np.full((new_t, new_s), fill_value=np.nan),
                      'waterLevel': np.full((new_t, new_s), fill_value=np.nan),
                      'stdWaterLevel': np.full((new_t, new_s), fill_value=np.nan),
                      'setup': np.full((new_t, new_s), fill_value=np.nan),
                      'runup2perc': dataDict['runup2perc'],
                      'runupMean': dataDict['runupMean'],
                      'qbx': np.full((new_t, new_s), fill_value=np.nan),
                      'qsx': np.full((new_t, new_s), fill_value=np.nan),
                      'qby': np.full((new_t, new_s), fill_value=np.nan),
                      'qsy': np.full((new_t, new_s), fill_value=np.nan),
                      'probabilitySuspension': np.full((new_t, new_s), fill_value=np.nan),
                      'probabilityMovement': np.full((new_t, new_s), fill_value=np.nan),
                      'suspendedSedVolume': np.full((new_t, new_s), fill_value=np.nan),
                      'bottomElevation': np.full((new_t, new_s), fill_value=np.nan),
                      'latitude': np.full((new_s), fill_value=np.nan),
                      'longitude': np.full((new_s), fill_value=np.nan),
                      'surveyNumber': dataDict['surveyNumber'],
                      'profileNumber': dataDict['profileNumber'],
                      'bathymetryDate': dataDict['bathymetryDate'],
                      'yFRF': dataDict['yFRF'],}

        if 'FIXED' in ofname:
            dataDict_n['bottomElevation'] = np.full((new_s), fill_value=np.nan)
        elif 'MOBILE' in ofname:
            dataDict_n['bottomElevation'] = np.full((new_t, new_s), fill_value=np.nan)
        else:
            print 'You need to modify makenc_CSHORErun in makenc.py to accept your new version name!'

        # find index of first point on dataDict grid
        min_x = min(dataDict['xFRF'])
        ind_minx = int(np.argwhere(dataDict_n['xFRF'] == min_x))
        max_x = max(dataDict['xFRF'])
        ind_maxx = int(np.argwhere(dataDict_n['xFRF'] == max_x))

        for ii in range(0, int(new_t)):
            dataDict_n['aveE'][ii][ind_maxx:ind_minx+1] = dataDict['aveE'][ii]
            dataDict_n['stdE'][ii][ind_maxx:ind_minx+1] = dataDict['stdE'][ii]
            dataDict_n['aveN'][ii][ind_maxx:ind_minx+1] = dataDict['aveN'][ii]
            dataDict_n['stdN'][ii][ind_maxx:ind_minx+1] = dataDict['stdN'][ii]
            dataDict_n['waveHs'][ii][ind_maxx:ind_minx+1] = dataDict['waveHs'][ii]
            dataDict_n['waveMeanDirection'][ii][ind_maxx:ind_minx+1] = dataDict['waveMeanDirection'][ii]
            dataDict_n['waterLevel'][ii][ind_maxx:ind_minx+1] = dataDict['waterLevel'][ii]
            dataDict_n['stdWaterLevel'][ii][ind_maxx:ind_minx+1] = dataDict['stdWaterLevel'][ii]
            dataDict_n['setup'][ii][ind_maxx:ind_minx+1] = dataDict['setup'][ii]
            dataDict_n['qbx'][ii][ind_maxx:ind_minx+1] = dataDict['qbx'][ii]
            dataDict_n['qsx'][ii][ind_maxx:ind_minx+1] = dataDict['qsx'][ii]
            dataDict_n['qby'][ii][ind_maxx:ind_minx+1] = dataDict['qby'][ii]
            dataDict_n['qsy'][ii][ind_maxx:ind_minx+1] = dataDict['qsy'][ii]
            dataDict_n['probabilitySuspension'][ii][ind_maxx:ind_minx+1] = dataDict['probabilitySuspension'][ii]
            dataDict_n['probabilityMovement'][ii][ind_maxx:ind_minx+1] = dataDict['probabilityMovement'][ii]
            dataDict_n['suspendedSedVolume'][ii][ind_maxx:ind_minx+1] = dataDict['suspendedSedVolume'][ii]
            dataDict_n['latitude'][ind_maxx:ind_minx+1] = dataDict['latitude'][ii]
            dataDict_n['longitude'][ind_maxx:ind_minx+1] = dataDict['longitude'][ii]


        if 'FIXED' in ofname:
            dataDict_n['bottomElevation'][ind_maxx:ind_minx + 1] = dataDict['bottomElevation']
        elif 'MOBILE' in ofname:
            for ii in range(0, int(new_t)):
                dataDict_n['bottomElevation'][ii][ind_maxx:ind_minx + 1] = dataDict['bottomElevation'][ii]
        else:
            print 'You need to modify makenc_CSHORErun in makenc.py to accept your new version name!'

    # get rid of all masks
    test = np.ma.masked_array(dataDict_n['aveE'], np.isnan(dataDict_n['aveE']))
    dataDict_n['aveE'] = test
    del test
    test = np.ma.masked_array(dataDict_n['stdE'], np.isnan(dataDict_n['stdE']))
    dataDict_n['stdE'] = test
    del test
    test = np.ma.masked_array(dataDict_n['aveN'], np.isnan(dataDict_n['aveN']))
    dataDict_n['aveN'] = test
    del test
    test = np.ma.masked_array(dataDict_n['stdN'], np.isnan(dataDict_n['stdN']))
    dataDict_n['stdN'] = test
    del test
    test = np.ma.masked_array(dataDict_n['waveHs'], np.isnan(dataDict_n['waveHs']))
    dataDict_n['waveHs'] = test
    del test
    test = np.ma.masked_array(dataDict_n['waveMeanDirection'], np.isnan(dataDict_n['waveMeanDirection']))
    dataDict_n['waveMeanDirection'] = test
    del test
    test = np.ma.masked_array(dataDict_n['waterLevel'], np.isnan(dataDict_n['waterLevel']))
    dataDict_n['waterLevel'] = test
    del test
    test = np.ma.masked_array(dataDict_n['stdWaterLevel'], np.isnan(dataDict_n['stdWaterLevel']))
    dataDict_n['stdWaterLevel'] = test
    del test
    test = np.ma.masked_array(dataDict_n['setup'], np.isnan(dataDict_n['setup']))
    dataDict_n['setup'] = test
    del test
    test = np.ma.masked_array(dataDict_n['qbx'], np.isnan(dataDict_n['qbx']))
    dataDict_n['qbx'] = test
    del test
    test = np.ma.masked_array(dataDict_n['qsx'], np.isnan(dataDict_n['qsx']))
    dataDict_n['qsx'] = test
    del test
    test = np.ma.masked_array(dataDict_n['qby'], np.isnan(dataDict_n['qby']))
    dataDict_n['qby'] = test
    del test
    test = np.ma.masked_array(dataDict_n['qsy'], np.isnan(dataDict_n['qsy']))
    dataDict_n['qsy'] = test
    del test
    test = np.ma.masked_array(dataDict_n['probabilitySuspension'], np.isnan(dataDict_n['probabilitySuspension']))
    dataDict_n['probabilitySuspension'] = test
    del test
    test = np.ma.masked_array(dataDict_n['probabilityMovement'], np.isnan(dataDict_n['probabilityMovement']))
    dataDict_n['probabilityMovement'] = test
    del test
    test = np.ma.masked_array(dataDict_n['suspendedSedVolume'], np.isnan(dataDict_n['suspendedSedVolume']))
    dataDict_n['suspendedSedVolume'] = test
    del test
    test = np.ma.masked_array(dataDict_n['latitude'], np.isnan(dataDict_n['latitude']))
    dataDict_n['latitude'] = test
    del test
    test = np.ma.masked_array(dataDict_n['longitude'], np.isnan(dataDict_n['longitude']))
    dataDict_n['longitude'] = test
    del test
    test = np.ma.masked_array(dataDict_n['bottomElevation'], np.isnan(dataDict_n['bottomElevation']))
    dataDict_n['bottomElevation'] = test
    del test

    # check to see if I screwed up!
    assert set(dataDict.keys()) == set(dataDict_n.keys()), 'You are missing dictionary keys in the new dictionary!'
    # replace the dictionary with the new dictionary
    del dataDict
    dataDict = dataDict_n
    del dataDict_n

    # now we flip everything that has a spatial dimension around so it will be all pretty like spicer wants?
    dataDict['aveN'] = np.flip(dataDict['aveN'], 1)
    dataDict['waveHs'] = np.flip(dataDict['waveHs'], 1)
    dataDict['aveE'] = np.flip(dataDict['aveE'], 1)
    dataDict['waveMeanDirection'] = np.flip(dataDict['waveMeanDirection'], 1)
    dataDict['stdWaterLevel'] = np.flip(dataDict['stdWaterLevel'], 1)
    dataDict['probabilitySuspension'] = np.flip(dataDict['probabilitySuspension'], 1)
    dataDict['stdN'] = np.flip(dataDict['stdN'], 1)
    dataDict['stdE'] = np.flip(dataDict['stdE'], 1)
    dataDict['bottomElevation'] = np.flip(dataDict['bottomElevation'], 1)
    dataDict['xFRF'] = np.flip(dataDict['xFRF'], 0)
    dataDict['qsy'] = np.flip(dataDict['qsy'], 1)
    dataDict['qsx'] = np.flip(dataDict['qsx'], 1)
    dataDict['waterLevel'] = np.flip(dataDict['waterLevel'], 1)
    dataDict['qbx'] = np.flip(dataDict['qbx'], 1)
    dataDict['qby'] = np.flip(dataDict['qby'], 1)
    dataDict['setup'] = np.flip(dataDict['setup'], 1)
    dataDict['longitude'] = np.flip(dataDict['longitude'], 0)
    dataDict['latitude'] = np.flip(dataDict['latitude'], 0)
    dataDict['suspendedSedVolume'] = np.flip(dataDict['suspendedSedVolume'], 1)
    dataDict['probabilityMovement'] = np.flip(dataDict['probabilityMovement'], 1)

    # write data to file
    write_data_to_nc(fid, varAtts, dataDict)
    # close file
    fid.close()

def makenc_intBATHY(ofname, dataDict, globalYaml, varYaml):
    """
    TODO: can this be combined with makenc_t0BATHY
    Args:
      ofname: this is the name of the cshore_ncfile you are building
      dataDict: keys must include... and match the varYaml file
        utmNorthing - this is utm in meters (not feet)

        utmEasting - this is utm in meters (not feet)
    
      globalYaml: yaml containing global meta data
      varYaml:  yaml containing variable meta data

    Returns:
      writes out the ncfile

    """

    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)

    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    # creating dimensions of data
    ni = fid.createDimension('ni', dataDict['utmEasting'].shape[1])
    nj = fid.createDimension('nj', dataDict['utmEasting'].shape[0])

    # write data to file
    write_data_to_nc(fid, varAtts, dataDict)
    # close file
    fid.close()

def makenc_t0BATHY(ofname, dataDict, globalYaml, varYaml):
    """# this is the script that builds the t0 netCDF file from the initial Bathy DEM (intBathy)

    Args:
      ofname (str): this is the name of the cshore_ncfile you are building
      dataDict (dict): keys must include... and matching keys to var Yaml
        xFRF - in m

        yFRF - in m

      globalYaml (str): CF compliant meta data in global file
      varYaml (str):  CF compliant variable meta data matching dataDict

    Returns:
      writes out the cshore_ncfile

    """

    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)

    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    # creating dimensions of data
    xFRF = fid.createDimension('xFRF', dataDict['xFRF'].shape[0])
    yFRF = fid.createDimension('yFRF', dataDict['yFRF'].shape[0])

    # write data to file
    write_data_to_nc(fid, varAtts, dataDict)
    # close file
    fid.close()

def makenc_tiBATHY(ofname, dataDict, globalYaml, varYaml):
    """# this is the script that builds the monthly ti netCDF file by incorporating the new survey data into the most recent Bathy DEM

    Args:
      ofname (str): this is the name of the cshore_ncfile you are building
      dataDict (str): keys must include... and matching keys to varYaml
            time:
            xFRF: - in m
            yFRF: - in m

      globalYaml (str): path to global yaml with CF compliant meta data
      varYaml (str): path to variable CF compliant meta data and matching the keys in data Dict

    Returns:
      writes out the netCDF file

    """

    globalAtts = import_template_file(globalYaml)
    varAtts = import_template_file(varYaml)

    # create netcdf file
    fid = init_nc_file(ofname, globalAtts)

    # creating dimensions of data
    time = fid.createDimension('time', dataDict['time'].shape[0])
    xFRF = fid.createDimension('xFRF', dataDict['xFRF'].shape[0])
    yFRF = fid.createDimension('yFRF', dataDict['yFRF'].shape[0])

    # write data to file
    write_data_to_nc(fid, varAtts, dataDict)
    # close file
    fid.close()