Skip to content

Commit

Permalink
Create RAM usage save mode.
Browse files Browse the repository at this point in the history
  • Loading branch information
@gnrgomes
gnrgomes committed Dec 15, 2023
1 parent c3a4a04 commit e4d79fa
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 11 deletions.
19 changes: 15 additions & 4 deletions src/lisfloodutilities/gridding/generate_grids.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,14 @@ def interpolation_mode_type(mode: str) -> str:
raise ArgumentTypeError(f'You must select a mode out of {list(Config.INTERPOLATION_MODES.keys())}.')
return mode

def memory_save_mode_type(mode: str) -> str:
if mode not in Config.MEMORY_SAVE_MODES:
raise ArgumentTypeError(f'You must select a RAM save mode out of {list(Config.MEMORY_SAVE_MODES.keys())}.')
return mode

def run(config_filename: str, infolder: str, output_file: str, processing_dates_file: str, file_utils: FileUtils,
output_tiff: bool, overwrite_output: bool, start_date: datetime = None, end_date: datetime = None,
interpolation_mode: str = 'adw', use_broadcasting: bool = False):
interpolation_mode: str = 'adw', use_broadcasting: bool = False, memory_save_mode: str = None):
"""
Interpolate text files containing (x, y, value) using inverse distance interpolation.
Produces as output, either a netCDF file containing all the grids or one TIFF file per grid.
Expand All @@ -53,7 +58,7 @@ def run(config_filename: str, infolder: str, output_file: str, processing_dates_
"""
global quiet_mode

conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode)
conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode, memory_save_mode)

if conf.start_date > conf.end_date:
raise ArgumentTypeError("Start date is greater than End date.")
Expand Down Expand Up @@ -135,7 +140,8 @@ def main(argv):
start_date='',
end_date=END_DATE_DEFAULT,
interpolation_mode='adw',
use_broadcasting=False)
use_broadcasting=False,
memory_save_mode='0')

parser.add_argument("-i", "--in", dest="infolder", required=True, type=FileUtils.folder_type,
help="Set input folder path with kiwis/point files",
Expand Down Expand Up @@ -169,6 +175,9 @@ def main(argv):
parser.add_argument("-m", "--mode", dest="interpolation_mode", required=False, type=interpolation_mode_type,
help="Set interpolation mode. [default: %(default)s]",
metavar=f"{list(Config.INTERPOLATION_MODES.keys())}")
parser.add_argument("-r", "--ramsavemode", dest="memory_save_mode", required=False, type=memory_save_mode_type,
help="Set memory save mode level. Used to reduce memory usage [default: %(default)s]",
metavar=f"{list(Config.MEMORY_SAVE_MODES.keys())}")
parser.add_argument("-b", "--broadcast", dest="use_broadcasting", action="store_true",
help="When set, computations will run faster in full broadcasting mode but require more memory. [default: %(default)s]")

Expand Down Expand Up @@ -206,6 +215,7 @@ def main(argv):
print_msg(f"Input Folder: {args.infolder}")
print_msg(f"Overwrite Output: {args.overwrite_output}")
print_msg(f"Interpolation Mode: {args.interpolation_mode}")
print_msg(f"RAM Save Mode: {args.memory_save_mode}")
print_msg(f"Broadcasting: {args.use_broadcasting}")
if args.out_tiff:
print_msg("Output Type: TIFF")
Expand All @@ -216,7 +226,8 @@ def main(argv):
print_msg(f"Config File: {config_filename}")

run(config_filename, args.infolder, args.output_file, args.processing_dates_file,
file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode, args.use_broadcasting)
file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode,
args.use_broadcasting, args.memory_save_mode)
except Exception as e:
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
Expand Down
24 changes: 17 additions & 7 deletions src/lisfloodutilities/gridding/lib/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,15 +157,18 @@ class Config(Printable):
# Each interpolation is paired with the number of neighbours used by the interpolation algorithm.
INTERPOLATION_MODES = {'nearest': 1, 'invdist': 20, 'adw': 11, 'cdd': 11,
'bilinear': 4, 'triangulation': 3, 'bilinear_delaunay': 4}
def __init__(self, config_filename: str, start_date: datetime = None,
end_date: datetime = None, quiet_mode: bool = False, interpolation_mode: str = 'adw'):
# Allows splitting the grid to interpolate into several parts and run isolated interpolations for each
MEMORY_SAVE_MODES = {'0': None, '1': 2, '2': 4, '3': 10, '4': 20, '5': 50}
def __init__(self, config_filename: str, start_date: datetime = None, end_date: datetime = None,
quiet_mode: bool = False, interpolation_mode: str = 'adw', memory_save_mode: str = '0'):
super().__init__(quiet_mode)
self.COLUMN_HEIGHT = "height"
self.COLUMN_VALUE = "value"
self.COLUMN_LON = "lon"
self.COLUMN_LAT = "lat"
self.VALUE_NAN = -9999.0
self.interpolation_mode = interpolation_mode
self.memory_save_mode = memory_save_mode
self.__setup_variable_config(config_filename)
self.__HEIGHT_CORRECTION_FACTOR = {"tx": 0.006, "tn": 0.006, "ta": 0.006, "ta6": 0.006, "pd": 0.00025}

Expand Down Expand Up @@ -330,6 +333,10 @@ def height_correction_factor(self) -> float:
def neighbours_near(self) -> int:
return self.scipy_modes_nnear[self.interpolation_mode]

@property
def num_of_splits(self) -> int:
return Config.MEMORY_SAVE_MODES[self.memory_save_mode]


class GriddingUtils(Printable):

Expand Down Expand Up @@ -407,6 +414,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
xp = np.array(x)
yp = np.array(y)
values = np.array(z)
df = None
if self.conf.interpolation_mode == 'cdd':
scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
self.conf.neighbours_near, mv_target, mv_source,
Expand All @@ -415,13 +423,15 @@ def generate_grid(self, filename: Path) -> np.ndarray:
cdd_map=self.conf.cdd_map,
cdd_mode=self.conf.cdd_mode,
cdd_options=self.conf.cdd_options,
use_broadcasting=self.use_broadcasting)
use_broadcasting=self.use_broadcasting,
num_of_splits=self.conf.num_of_splits)
else:
scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
self.conf.neighbours_near, mv_target, mv_source,
target_is_rotated=False, parallel=False,
mode=self.conf.interpolation_mode,
use_broadcasting=self.use_broadcasting)
use_broadcasting=self.use_broadcasting,
num_of_splits=self.conf.num_of_splits)

# reset search radius to a fixed value
scipy_interpolation.min_upper_bound = self.conf.min_upper_bound
Expand All @@ -440,7 +450,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
# CSV_DELIMITER = '\t'
# FILES_WILDCARD = '??????????00_all.kiwis'
#
# def __init__(self, conf: Config, overwrite_file: bool = False, infolder: Path, quiet_mode: bool = False):
# def __init__(self, conf: Config, infolder: Path, overwrite_file: bool = False, quiet_mode: bool = False):
# super().__init__(quiet_mode)
# self.conf = conf
# self.overwrite_file = overwrite_file
Expand Down Expand Up @@ -469,7 +479,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
# self.file_group_read_idx += 1
# raise StopIteration
#
# def __get_filter_classes(self) -> array:
# def __get_filter_classes(self) -> list:
# '''
# TODO: Implement the class.
# '''
Expand Down Expand Up @@ -534,4 +544,4 @@ def generate_grid(self, filename: Path) -> np.ndarray:
#
# def __processable_file(self, file_timestamp: datetime, start_date: datetime = None, end_date: datetime = None) -> bool:
# return (start_date is not None and start_date <= file_timestamp and
# end_date is not None and file_timestamp <= end_date)
# end_date is not None and file_timestamp <= end_date)

0 comments on commit e4d79fa

Please sign in to comment.