From cce4b99c7046217b1ec1192118a786636e0d8e54 Mon Sep 17 00:00:00 2001 From: Steve Wardle Date: Fri, 6 May 2022 15:26:41 +0100 Subject: [PATCH] Import @109242 2022.05.1 --- admin/install_mule.sh | 43 +-- mule/LICENCE.txt | 2 +- mule/docs/source/conf.py | 6 +- mule/lib/mule/__init__.py | 2 +- mule/lib/mule/packing.py | 3 +- um_packing/setup.py | 25 +- um_ppibm/setup.py | 22 +- um_spiral_search/setup.py | 25 +- um_sstpert/setup.py | 24 +- um_utils/docs/source/index.rst | 2 +- um_utils/docs/source/um_utils/editmask.rst | 110 ++++++++ um_utils/lib/um_utils/convpp.py | 18 ++ um_utils/lib/um_utils/editmask.py | 291 +++++++++++++++++++++ um_utils/setup.py | 5 +- um_wafccb/setup.py | 21 +- 15 files changed, 504 insertions(+), 95 deletions(-) create mode 100644 um_utils/docs/source/um_utils/editmask.rst create mode 100644 um_utils/lib/um_utils/editmask.py diff --git a/admin/install_mule.sh b/admin/install_mule.sh index 845ed6e..3d32545 100755 --- a/admin/install_mule.sh +++ b/admin/install_mule.sh @@ -7,10 +7,10 @@ # # Mule, UM packing extension and UM library installation script # -# In most cases the modules can be directly installed via the usual -# python setuptools methods. However in some cases this might not be +# In most cases the modules can be directly installed via the usual +# python setuptools methods. However in some cases this might not be # possible, so this script instead builds all modules to a dummy -# install location in a temporary directory and then copies the +# install location in a temporary directory and then copies the # results to the chosen destinations. # set -eu @@ -86,9 +86,9 @@ while [ $# -gt 3 ] ; do SSTPERT_LIB=$1 ;; --wafccb_lib) shift WAFCCB_LIB=$1 ;; - --spiral_lib) + --spiral_lib) SPIRAL_LIB="build" ;; - --ppibm_lib) + --ppibm_lib) PPIBM_LIB="build" ;; --library_lock) LIBRARY_LOCK="lock" ;; @@ -116,8 +116,13 @@ if [ -n "$PPIBM_LIB" ] ; then MODULE_LIST="$MODULE_LIST um_ppibm" fi -# A few hardcoded settings -PYTHONEXEC=${PYTHONEXEC:-python2.7} +# Find out the version of the current interpreter, since this is what we +# will be trying to install against +PYTHONVER=$(python -c "from platform import python_version ; print(python_version())") +PYTHONEXEC=python$(cut -d . -f-2 <<< $PYTHONVER) +echo "Installing against Python $PYTHONVER" + +# Setup a temporary directory where the install will be initially created SCRATCHDIR=$(mktemp -d) SCRATCHLIB=$SCRATCHDIR/lib/$PYTHONEXEC/site-packages @@ -129,7 +134,7 @@ if [ ! ${BIN_DEST:0:1} == "/" ] ; then BIN_DEST=$PWD/$BIN_DEST fi -# Create install directores - they may already exist but should be +# Create install directores - they may already exist but should be # empty if they do, also check the modules exist in the cwd exit=0 for module in $MODULE_LIST ; do @@ -176,7 +181,7 @@ if [ -n "$WAFCCB_LIB" ] && [ ! -d $WAFCCB_LIB ] ; then fi # Make a temporary directory to hold the installs -mkdir -p $SCRATCHLIB +mkdir -p $SCRATCHLIB ln -s $SCRATCHDIR/lib $SCRATCHDIR/lib64 # The install command will complain if this directory isn't on the path @@ -196,7 +201,7 @@ wc_root=$(pwd) # use ln -r for this or realpath but since they aren't as common as we'd # like let's use Python function pyrelpath(){ - $PYTHONEXEC -c "import os.path; print(os.path.relpath(\"$1\",\"$2\"))" + python -c "import os.path; print(os.path.relpath(\"$1\",\"$2\"))" } # Packing library first @@ -214,7 +219,7 @@ else fi echo "Building packing module..." -$PYTHONEXEC setup.py build_ext --inplace \ +python setup.py build_ext --inplace \ -I$SHUMLIB/include -L$SHUMLIB/lib -R$rpath # SSTPert library (if being used) @@ -235,7 +240,7 @@ if [ -n "$SSTPERT_LIB" ] ; then fi echo "Building sstpert module..." - $PYTHONEXEC setup.py build_ext --inplace \ + python setup.py build_ext --inplace \ -I$SSTPERT_LIB/include \ -L$SSTPERT_LIB/lib:$SHUMLIB/lib \ -R$rpath @@ -257,7 +262,7 @@ if [ -n "$WAFCCB_LIB" ] ; then fi echo "Building wafccb module..." - $PYTHONEXEC setup.py build_ext --inplace \ + python setup.py build_ext --inplace \ -I$WAFCCB_LIB/include \ -L$WAFCCB_LIB/lib \ -R$rpath @@ -279,7 +284,7 @@ if [ -n "$SPIRAL_LIB" ] ; then fi echo "Building spiral search module..." - $PYTHONEXEC setup.py build_ext --inplace \ + python setup.py build_ext --inplace \ -I$SHUMLIB/include \ -L$SHUMLIB/lib \ -R$rpath @@ -301,7 +306,7 @@ if [ -n "$PPIBM_LIB" ] ; then fi echo "Building ppibm module..." - $PYTHONEXEC setup.py build_ext --inplace \ + python setup.py build_ext --inplace \ -I$SHUMLIB/include \ -L$SHUMLIB/lib \ -R$rpath @@ -316,7 +321,7 @@ function install(){ cd $wc_root/$module echo "Installing $module module to $SCRATCHDIR" - $PYTHONEXEC setup.py install --prefix $SCRATCHDIR + python setup.py install --prefix $SCRATCHDIR } for module in $MODULE_LIST ; do @@ -336,7 +341,7 @@ function unpack_and_copy(){ unzip_dir=$SCRATCHLIB/${module}_unzipped_egg unzip $egg -d $unzip_dir egg=$unzip_dir - fi + fi destdir=$LIB_DEST/$module echo "Installing $module to $destdir" @@ -346,7 +351,7 @@ function unpack_and_copy(){ # For the execs, also copy these to the bin directory if [ $module == "um_utils" ] || [ $module == "um_sstpert" ] ; then echo "Installing $module execs and info to $BIN_DEST/" - cp -vr $egg/EGG-INFO $BIN_DEST/$module.egg-info + cp -vr $egg/EGG-INFO $BIN_DEST/$module.egg-info cp -vr $SCRATCHDIR/bin/* $BIN_DEST/ fi } @@ -364,7 +369,7 @@ function cleanup(){ cd $wc_root/$module echo "Cleaning $module module" - $PYTHONEXEC setup.py clean + python setup.py clean } for module in $MODULE_LIST ; do diff --git a/mule/LICENCE.txt b/mule/LICENCE.txt index da7d6d5..642729b 100644 --- a/mule/LICENCE.txt +++ b/mule/LICENCE.txt @@ -1,5 +1,5 @@ !-------------------------------------------------------------------------------! -! (C) Crown Copyright 2020, Met Office. All rights reserved. ! +! (C) Crown Copyright 2022, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/mule/docs/source/conf.py b/mule/docs/source/conf.py index 2e7c457..3c6da0b 100644 --- a/mule/docs/source/conf.py +++ b/mule/docs/source/conf.py @@ -48,16 +48,16 @@ # General information about the project. project = u'Mule' -copyright = u'2020, UM Systems Team' +copyright = u'2022, UM Systems Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2020.01.1' +version = '2022.05.1' # The full version, including alpha/beta/rc tags. -release = '2020.01.1' +release = '2022.05.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/mule/lib/mule/__init__.py b/mule/lib/mule/__init__.py index e617fa4..7a5287c 100644 --- a/mule/lib/mule/__init__.py +++ b/mule/lib/mule/__init__.py @@ -57,7 +57,7 @@ from contextlib import contextmanager from mule.stashmaster import STASHmaster -__version__ = "2020.01.1" +__version__ = "2022.05.1" # UM fixed length header names and positions _UM_FIXED_LENGTH_HEADER = [ diff --git a/mule/lib/mule/packing.py b/mule/lib/mule/packing.py index 6696fad..b28d239 100644 --- a/mule/lib/mule/packing.py +++ b/mule/lib/mule/packing.py @@ -156,7 +156,8 @@ def _wgdos_pack_field(data, mdi, acc): packed byte data. """ - data_bytes = mo_pack.compress_wgdos(data.astype("f4"), acc, mdi) + data_buffer = mo_pack.compress_wgdos(data.astype("f4"), acc, mdi) + data_bytes = data_buffer.tobytes() return data_bytes except ImportError as err: diff --git a/um_packing/setup.py b/um_packing/setup.py index d426762..2293a6f 100644 --- a/um_packing/setup.py +++ b/um_packing/setup.py @@ -49,9 +49,10 @@ def run(self): elif os.path.isdir(cleanpath[0]): shutil.rmtree(cleanpath[0]) + setuptools.setup( name='um_packing', - version='2020.01.1', + version='2022.05.1', description='Unified Model packing library extension', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', @@ -59,16 +60,12 @@ def run(self): package_dir={'': 'lib'}, packages=['um_packing', 'um_packing.tests'], - features={ - 'packing': setuptools.Feature( - "SHUMlib Packing library", - standard=True, - ext_modules=[ - setuptools.Extension( - 'um_packing.um_packing', - ['lib/um_packing/um_packing.c'], - include_dirs=[np.get_include()], - libraries=["shum_byteswap", - "shum_wgdos_packing", - "shum_string_conv", - "shum_constants"])])}) + ext_modules=[ + setuptools.Extension( + 'um_packing.um_packing', + ['lib/um_packing/um_packing.c'], + include_dirs=[np.get_include()], + libraries=["shum_byteswap", + "shum_wgdos_packing", + "shum_string_conv", + "shum_constants"])]) diff --git a/um_ppibm/setup.py b/um_ppibm/setup.py index 6dcd598..f0ac371 100644 --- a/um_ppibm/setup.py +++ b/um_ppibm/setup.py @@ -52,22 +52,18 @@ def run(self): setuptools.setup( name='um_ppibm', - version='2020.01.1', + version='2022.05.1', description='Unified Model pp conversion utility with IBM number format', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', cmdclass={'clean': CleanCommand}, package_dir={'': 'lib'}, packages=['um_ppibm', ], - features={ - 'ieee2ibm32': setuptools.Feature( - "SHUMlib IBM conversion routine", - standard=True, - ext_modules=[ - setuptools.Extension( - 'um_ppibm.um_ieee2ibm32', - ['lib/um_ppibm/um_ieee2ibm32.c'], - include_dirs=[np.get_include()], - libraries=["shum_string_conv", - "shum_byteswap", - "shum_data_conv"])])}) + ext_modules=[ + setuptools.Extension( + 'um_ppibm.um_ieee2ibm32', + ['lib/um_ppibm/um_ieee2ibm32.c'], + include_dirs=[np.get_include()], + libraries=["shum_string_conv", + "shum_byteswap", + "shum_data_conv"])]) diff --git a/um_spiral_search/setup.py b/um_spiral_search/setup.py index 9168abb..d9839c4 100644 --- a/um_spiral_search/setup.py +++ b/um_spiral_search/setup.py @@ -49,9 +49,10 @@ def run(self): elif os.path.isdir(cleanpath[0]): shutil.rmtree(cleanpath[0]) + setuptools.setup( name='um_spiral_search', - version='2020.01.1', + version='2022.05.1', description='Unified Model Spiral Search extension', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', @@ -59,16 +60,12 @@ def run(self): package_dir={'': 'lib'}, packages=['um_spiral_search', 'um_spiral_search.tests'], - features={ - 'spiralsearch': setuptools.Feature( - "UM Spiral Search library", - standard=True, - ext_modules=[ - setuptools.Extension( - 'um_spiral_search.um_spiral_search', - ['lib/um_spiral_search/um_spiral_search.c'], - include_dirs=[np.get_include()], - libraries=["shum_spiral_search", - "shum_string_conv", - "shum_constants"]) - ])}) + ext_modules=[ + setuptools.Extension( + 'um_spiral_search.um_spiral_search', + ['lib/um_spiral_search/um_spiral_search.c'], + include_dirs=[np.get_include()], + libraries=["shum_spiral_search", + "shum_string_conv", + "shum_constants"]) + ]) diff --git a/um_sstpert/setup.py b/um_sstpert/setup.py index 480d6a0..e88c3b6 100644 --- a/um_sstpert/setup.py +++ b/um_sstpert/setup.py @@ -49,28 +49,24 @@ def run(self): elif os.path.isdir(cleanpath[0]): shutil.rmtree(cleanpath[0]) + setuptools.setup( name='um_sstpert', - version='2020.01.1', + version='2022.05.1', description='Unified Model SST-perturbation extension and utility', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', cmdclass={'clean': CleanCommand}, package_dir={'': 'lib'}, packages=['um_sstpert'], - features={ - 'sstpert': setuptools.Feature( - "UM SST-pert library (requires UM Licence)", - standard=True, - ext_modules=[ - setuptools.Extension( - 'um_sstpert.um_sstpert', - ['lib/um_sstpert/um_sstpert.c'], - include_dirs=[np.get_include()], - libraries=["um_sstpert", - "shum_string_conv", - "shum_constants"]) - ])}, + ext_modules=[ + setuptools.Extension( + 'um_sstpert.um_sstpert', + ['lib/um_sstpert/um_sstpert.c'], + include_dirs=[np.get_include()], + libraries=["um_sstpert", + "shum_string_conv", + "shum_constants"])], entry_points={ 'console_scripts': [ 'mule-sstpert = um_sstpert:_main', diff --git a/um_utils/docs/source/index.rst b/um_utils/docs/source/index.rst index 1411312..ef1ddb1 100644 --- a/um_utils/docs/source/index.rst +++ b/um_utils/docs/source/index.rst @@ -20,6 +20,7 @@ This is the documentation for the UM utilities, which use the Mule API. um_utils/version um_utils/unpack um_utils/convpp + um_utils/editmask Indices and tables ================== @@ -27,4 +28,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/um_utils/docs/source/um_utils/editmask.rst b/um_utils/docs/source/um_utils/editmask.rst new file mode 100644 index 0000000..48e878a --- /dev/null +++ b/um_utils/docs/source/um_utils/editmask.rst @@ -0,0 +1,110 @@ +Edit-mask (Utility for manual editing of land/sea mask) +======================================================= + +This utility is used to enable the manual editing of a land/sea mask from +a UM ancil file. Rather than designing a full GUI utility the intention is +that the tool allows the land/sea mask data to be exported as an image file +and edited externally using a program of the users choice. Once edited the +image file may be ingested back into a new UM ancil file. + +For the image editing, one has to be careful about how the file is saved to +ensure it is still compatible with the format expected by PIL (Python Imaging +Library). It has been tested using the free utility GIMP +(GNU Image Manipulation Program). After loading the file make careful edits +using only "hard edged" tools (i.e. pixel brush drawing tools) so as to ensure +the image remains purely black & white. Use the save option to "overwrite" the +original file for best results as it will avoid altering the format too much. + + +Command line utility +-------------------- +Here is the help text for the command line utility (obtainable by running +``mule-editmask --help``): + +.. code-block:: none + + ========================================================================== + * EDIT MASK - Editor for Land Sea Masks in UM Files (using the Mule API) * + ========================================================================== + usage: + mule-editmask [-h] {genimage,genancil} [options] + + Description + + positional arguments: + {genimage,genancil} + genimage generate image file (run "mule-editmask genimage --help" + for specific help on this command) + + genancil generate ancil file (run "mule-editmask genancil --help" + for specific help on this command) + + optional arguments: + -h, --help show this help message and exit + +Note that as shown above the command has two modes of operation - ``genimage`` +and ``genancil``, with each requiring different arguments. Here are the help +texts for these two modes: + +(from ``mule-editmask genimage --help``): + +.. code-block:: none + + ========================================================================== + * EDIT MASK - Editor for Land Sea Masks in UM Files (using the Mule API) * + ========================================================================== + usage: + mule-editmask genimage [-h] ancil_file image_file + + This will generate an image file which represents the contents + of the mask from the input file (ready for editing) + + positional arguments: + ancil_file UM Ancillary File containing source mask + + image_file Filename for output image (.png will be appended) + + + optional arguments: + -h, --help show this help message and exit + + +(from ``mule-editmask genancil --help``): + +.. code-block:: none + + ========================================================================== + * EDIT MASK - Editor for Land Sea Masks in UM Files (using the Mule API) * + ========================================================================== + usage: + mule-editmask genancil [-h] + [--text-file text_file] ancil_file mask_file output_file + + This will create a copy of the original file but with the data + from the mask image inserted into the mask field + + positional arguments: + ancil_file UM Ancillary File containing original mask + + mask_file Filename of edited mask, either: + A png image produced by 'genimage' + and edited externally + OR + A txt file produced by a previous + invocation of 'genancil' (see text_file) + output_file Filename for Ancillary File with new mask data + + optional arguments: + -h, --help show this help message and exit + --text-file TEXT_FILE + Filename for text file which will contain a list + of the changed points, and can be used as the + 'mask_file' argument in future calls if needed + +um_utils.editmask API +--------------------- +Here is the API documentation (auto-generated): + +.. automodule:: um_utils.editmask + :members: + :show-inheritance: diff --git a/um_utils/lib/um_utils/convpp.py b/um_utils/lib/um_utils/convpp.py index 842deda..c742ee6 100644 --- a/um_utils/lib/um_utils/convpp.py +++ b/um_utils/lib/um_utils/convpp.py @@ -26,12 +26,14 @@ """ import sys +import re import mule import mule.pp import argparse import textwrap from um_utils.pumf import pprint, _banner from um_utils.version import report_modules +import mule.stashmaster # See if the IBM conversion module is available try: @@ -105,6 +107,14 @@ def _main(): '--keep_addressing', "-k", action='store_true', help="Don't modify address elements LBNREC, LBEGIN and LBUSER(2)\n" "(might be required for legacy compatability)") + parser.add_argument( + "--stashmaster", + help="either the full path to a valid stashmaster file, or a UM \n" + "version number e.g. '10.2'; if given a number convpp will look in \n" + "the path defined by: \n" + " mule.stashmaster.STASHMASTER_PATH_PATTERN \n" + "which by default is : \n" + " $UMDIR/vnX.X/ctldata/STASHmaster/STASHmaster_A\n") # If the user supplied no arguments, print the help text and exit if len(sys.argv) == 1: @@ -123,6 +133,14 @@ def _main(): if mule.pp.file_is_pp_file(input_file): raise ValueError("Input file should be a UM File, not a PP file") + if args.stashmaster is not None: + if re.match(r"\d+.\d+", args.stashmaster): + mule.stashmaster.STASHMASTER_PATH_PATTERN = ( + "$UMDIR/vn{0}/ctldata/STASHmaster/STASHmaster_A" + .format(args.stashmaster)) + else: + mule.stashmaster.STASHMASTER_PATH_PATTERN = args.stashmaster + um_file = mule.load_umfile(input_file) if um_file.fixed_length_header.dataset_type not in (1, 2, 3, 4): diff --git a/um_utils/lib/um_utils/editmask.py b/um_utils/lib/um_utils/editmask.py new file mode 100644 index 0000000..765a535 --- /dev/null +++ b/um_utils/lib/um_utils/editmask.py @@ -0,0 +1,291 @@ +# *****************************COPYRIGHT****************************** +# (C) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file LICENCE.txt +# which you should have received as part of this distribution. +# *****************************COPYRIGHT****************************** +# +# This file is part of the UM utilities module, which use the Mule API. +# +# Mule and these utilities are free software: you can redistribute it and/or +# modify them under the terms of the Modified BSD License, as published by the +# Open Source Initiative. +# +# These utilities are distributed in the hope that they will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Modified BSD License for more details. +# +# You should have received a copy of the Modified BSD License +# along with these utilities. +# If not, see . +""" +EDITMASK is a utility to assist in manual editing of Land Sea Masks in +UM files. Rather than providing a GUI or direct editing capability +it converts the data into an image file for external editing, and then +allows the result to be placed back into a UM ancil. + +""" +import sys +import os +import mule +import numpy as np +import argparse +import textwrap +import warnings +from PIL import Image + +from um_utils.version import report_modules +from um_utils.pumf import pprint, PRINT_SETTINGS + +# Set pumf's pprinter to only print headers as we don't care +# about the max/min of the field +PRINT_SETTINGS["headers_only"] = True + +# Suppress the warning about the file version in ancil files +# as the STASHmaster isn't needed or relevant for this utility +warnings.filterwarnings( + "ignore", + r".*Ancillary files do not define the UM version number.*") + + +def _banner(message, banner_char="%"): + """A simple function which returns a banner string.""" + return "{0:s}\n* {1:s} *\n{0:s}\n".format( + banner_char*(len(message)+4), message) + + +def genimage(ancil_file, image_file): + """ + Generate a PNG image of the land sea mask found in + the given file, ready for editing with an external + program of the user's choice. + + """ + # Find the land sea mask in the file + anc = mule.AncilFile.from_file(ancil_file) + lsm_data = None + for field in anc.fields: + if field.lbuser4 == 30: + lsm_data = field.get_data() + break + if lsm_data is None: + raise ValueError("No Land/Sea mask found in file") + + # Scale the 0.0 - 1.0 data to run from 0.0 to 255.0 + lsm_data = lsm_data*255.0 + + # Convert to an image, flip the array so that it appears + # oriented properly north-south as on typical maps etc + img = Image.fromarray(lsm_data[::-1, :].astype(np.uint8)) + + # And output as a PNG file (add the extension if the user + # didn't to begin with) + if not image_file.endswith(".png"): + image_file += ".png" + img.save(image_file) + + +def genancil(ancil_file, mask_file, output_file, text_file): + """ + Generate a copy of the original ancil file using the mask + data from either an edited image, or a text file produced + by an earlier run of this method + + """ + # Load the ancillary file we are going to use and find the + # land sea mask in it + anc = mule.AncilFile.from_file(ancil_file) + lsm_field = None + for field in anc.fields: + if field.lbuser4 == 30: + lsm_field = field + break + if lsm_field is None: + raise ValueError("No Land/Sea mask found in file") + + # Get the existing field data + field_data = field.get_data() + + # The mask file could be an image or a text file + if mask_file.endswith(".png"): + # Open the image file, and convert the data inside into + # an array that will fit back into the field (it has to + # be flipped from its visual orientation) + img = Image.open(mask_file) + lsm_data = (np.array(img)[::-1, :] / 255).astype(">i8") + elif mask_file.endswith(".txt"): + lsm_data = field_data.copy() + # Read the mask file + with open(mask_file, "r") as mask: + for line in mask.readlines(): + line = line.strip() + # Skip comments/blanks + if line.startswith("#") or line == "": + continue + # If the line is one of the lookup headers + # at the top of the file, check it against + # the lookup of the ancil field + if line.startswith("("): + _, lookup, _, text_value = line.split() + ancil_value = str(getattr(lsm_field, lookup)) + if ancil_value != text_value: + msg = ("Lookup headers don't agree.\n" + " Text file : {0} = {1}\n" + " Ancil file: {0} = {2}").format( + lookup, text_value, ancil_value + ) + warnings.warn_explicit(msg) + continue + x, y, diff = map(int, line.strip().split()) + lsm_data[x, y] = lsm_data[x, y] - diff + else: + raise ValueError( + "Modification data should be either an image or \n" + "a text file") + + # Check to make sure this array will fit into the field + # (it should do provided the user has provided the same + # file as they used to generate the image) + if lsm_data.shape != field_data.shape: + raise ValueError( + "Image Land/Sea mask is not the same shape as " + "the mask found in the file") + + # If requested, write the difference file to a text file + if text_file is not None: + # Calculate the difference between the two fields + diff_data = field_data - lsm_data + + # Write the points which have changed to a file + if not text_file.endswith(".txt"): + text_file += ".txt" + with open(text_file, "w") as text: + text.write( + "# EDIT MASK: This file can be used to provide \n" + "# the tool with a listing of which points to change. \n\n" + "# Lookup headers of lsm field: \n") + pprint(lsm_field, text) + text.write( + "# Changed points; Pairs of X + Y indices, \n" + "# plus indicator of the type of change: \n" + "# 1 == Previously land, now sea \n" + "# -1 == Previously sea, now land \n") + + for points in zip(*np.where(diff_data != 0.0)): + text.write(" {0} {1} {2}\n".format(points[0], + points[1], + diff_data[points])) + + # Populate the field using the data + provider = mule.ArrayDataProvider(lsm_data) + lsm_field.set_data_provider(provider) + + # And finall write out the finished file + anc.to_file(output_file) + + +def _main(): + """ + Main function; accepts command line arguments. + + """ + # Setup help text + help_prolog = """ usage: + %(prog)s [-h] {genimage,genancil} [options] + + Description + """ + title = _banner( + "EDIT MASK - Editor for Land Sea Masks in UM Files " + "(using the Mule API)", banner_char="=") + + # Setup the parser + parser = argparse.ArgumentParser( + usage=argparse.SUPPRESS, + description=title + textwrap.dedent(help_prolog), + formatter_class=argparse.RawTextHelpFormatter, + ) + + # The command has multiple forms depending on what the user is doing + subparsers = parser.add_subparsers(dest='command') + + # Options for generating image file for editing + sub_prolog = """ usage: + {0} genimage [-h] ancil_file image_file + + This will generate an image file which represents the contents + of the mask from the input file (ready for editing) + """.format(os.path.basename(sys.argv[0])) + + parser_image = subparsers.add_parser( + "genimage", formatter_class=argparse.RawTextHelpFormatter, + description=title + textwrap.dedent(sub_prolog), + usage=argparse.SUPPRESS, + help="generate image file (run \"%(prog)s genimage --help\" \n" + "for specific help on this command)\n ") + + parser_image.add_argument( + "ancil_file", + help="UM Ancillary File containing source mask\n ") + parser_image.add_argument( + "image_file", + help="Filename for output image (.png will be appended)\n ") + + # Options for generating new mask file from image + sub_prolog = """ usage: + {0} genancil [-h] + [--text-file text_file] ancil_file mask_file output_file + + This will create a copy of the original file but with the data + from the mask image inserted into the mask field + """.format(os.path.basename(sys.argv[0])) + + parser_ancil = subparsers.add_parser( + "genancil", formatter_class=argparse.RawTextHelpFormatter, + description=title + textwrap.dedent(sub_prolog), + usage=argparse.SUPPRESS, + help="generate ancil file (run \"%(prog)s genancil --help\" \n" + "for specific help on this command)\n") + + parser_ancil.add_argument( + "--text-file", + help="Filename for text file which will contain a list \n" + "of the changed points, and can be used as the \n" + "'mask_file' argument in future calls if needed\n", + required=False, action="store") + + parser_ancil.add_argument( + "ancil_file", + help="UM Ancillary File containing original mask\n ") + parser_ancil.add_argument( + "mask_file", + help="Filename of edited mask, either: \n" + " A png image produced by 'genimage' \n" + " and edited externally\n " + "OR \n" + " A txt file produced by a previous \n" + " invocation of 'genancil' (see text_file)\n") + parser_ancil.add_argument( + "output_file", + help="Filename for Ancillary File with new mask data\n") + + # If the user supplied no arguments, print the help text and exit + if len(sys.argv) == 1: + parser.print_help() + parser.exit(1) + + args = parser.parse_args() + + # Print version information + print(_banner("(EDIT-MASK) Module Information")), + report_modules() + + if args.command == "genimage": + genimage(args.ancil_file, args.image_file) + elif args.command == "genancil": + genancil(args.ancil_file, args.mask_file, + args.output_file, args.text_file) + + +if __name__ == "__main__": + _main() diff --git a/um_utils/setup.py b/um_utils/setup.py index b5b5950..1198e6f 100644 --- a/um_utils/setup.py +++ b/um_utils/setup.py @@ -47,7 +47,7 @@ def run(self): setuptools.setup( name='um_utils', - version='2020.01.1', + version='2022.05.1', description='Unified Model Fields File utilities', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', @@ -79,4 +79,5 @@ def run(self): 'mule-fixframe = um_utils.fixframe:_main', 'mule-unpack = um_utils.unpack:_main', 'mule-select = um_utils.select:_main', - 'mule-convpp = um_utils.convpp:_main']}) + 'mule-convpp = um_utils.convpp:_main', + 'mule-editmask = um_utils.editmask:_main']}) diff --git a/um_wafccb/setup.py b/um_wafccb/setup.py index b31c8f0..d265e22 100644 --- a/um_wafccb/setup.py +++ b/um_wafccb/setup.py @@ -49,23 +49,20 @@ def run(self): elif os.path.isdir(cleanpath[0]): shutil.rmtree(cleanpath[0]) + setuptools.setup( name='um_wafccb', - version='2020.01.1', + version='2022.05.1', description='Unified Model WAFC CB extension', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', cmdclass={'clean': CleanCommand}, package_dir={'': 'lib'}, packages=['um_wafccb'], - features={ - 'wafccb': setuptools.Feature( - "UM WAFC CB library (requires UM Licence)", - standard=True, - ext_modules=[ - setuptools.Extension( - 'um_wafccb.um_wafccb', - ['lib/um_wafccb/um_wafccb.c'], - include_dirs=[np.get_include()], - libraries=["um_wafccb"]), - ])}) + ext_modules=[ + setuptools.Extension( + 'um_wafccb.um_wafccb', + ['lib/um_wafccb/um_wafccb.c'], + include_dirs=[np.get_include()], + libraries=["um_wafccb"]), + ])