diff --git a/bin/rios_subproc.py b/bin/rios_subproc.py deleted file mode 100755 index 84730aa0..00000000 --- a/bin/rios_subproc.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -""" -Main program for RIOS subprocesses. - -""" -# This file is part of RIOS - Raster I/O Simplification -# Copyright (C) 2012 Sam Gillingham, Neil Flood -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from __future__ import print_function - -from rios.parallel import subproc - -import sys - -if __name__ == "__main__": - nArgs = len(sys.argv) - 1 - - # This is for the case of no commandline args - if sys.version_info[0] > 2: - # For Python 3, use binary buffer objects - # otherwise unpickling fails - inf = sys.stdin.buffer - outf = sys.stdout.buffer - else: - inf = sys.stdin - outf = sys.stdout - - # These are for the cases with some commandline args - inFileName = None - if nArgs >= 1: - inFileName = sys.argv[1] - inf = open(inFileName, 'rb') - if nArgs == 2: - outf = open(sys.argv[2], 'wb') - - subproc.runJob(inf, outf, inFileName) diff --git a/bin/rios_subproc_awsbatch.py b/bin/rios_subproc_awsbatch.py deleted file mode 100755 index 7991bc76..00000000 --- a/bin/rios_subproc_awsbatch.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python -""" -Main program for RIOS subprocesses invoked via AWS Batch. - -""" -# This file is part of RIOS - Raster I/O Simplification -# Copyright (C) 2012 Sam Gillingham, Neil Flood -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from __future__ import print_function - -import os -import io -import time -import boto3 - -from rios.parallel import subproc - -# These vars are set in the container environment -# by CloudFormation -BUCKET = os.getenv("RIOSBucket") -INQUEUE = os.getenv("RIOSInQueue") -OUTQUEUE = os.getenv("RIOSOutQueue") -DFLT_NOMSG_TIMEOUT_SECS = 60 * 60 # 1 hour -NOMSG_TIMEOUT_SECS = int(os.getenv('RIOS_NOMSG_TIMEOUT', - default=DFLT_NOMSG_TIMEOUT_SECS)) - -# keep a track of the last time we got a message -# - if too long we can assume the main script exited -# and exit ourselves -LAST_MESSAGE_TIME = time.time() - - -def main(): - - global LAST_MESSAGE_TIME - - s3Client = boto3.client('s3') - # SQS client needs a region - should be same as s3 bucket - response = s3Client.get_bucket_location(Bucket=BUCKET) - region = response['LocationConstraint'] - sqsClient = boto3.client('sqs', region_name=region) - - while True: - # get a message from the queue - resp = sqsClient.receive_message(QueueUrl=INQUEUE, - MaxNumberOfMessages=1, WaitTimeSeconds=20) # must be <= 20 - if 'Messages' in resp and len(resp['Messages']) > 0: - # we got something - LAST_MESSAGE_TIME = time.time() - - # just look at the first one (just asked for 1) - msg = resp['Messages'][0] - body = msg['Body'] - receiptHandle = msg['ReceiptHandle'] - sqsClient.delete_message( - QueueUrl=INQUEUE, ReceiptHandle=receiptHandle) - - # message from the main script to exit - if body == 'Stop': - print('Job Exiting') - break - - print('Started', body) - - # get the info out of the pkl filename - bl, x, y, o = body.split('_') - outfile = 'block_{}_{}_out.pkl'.format(x, y) - - # read the input pkl - inPkl = io.BytesIO() - s3Client.download_fileobj(BUCKET, body, inPkl) - inPkl.seek(0) - - # delete it - s3Client.delete_object(Bucket=BUCKET, Key=body) - - # run the job - outPkl = io.BytesIO() - subproc.runJob(inPkl, outPkl) - - # upload the result - outPkl.seek(0) - s3Client.upload_fileobj(outPkl, BUCKET, outfile) - - # send message back to main script - sqsClient.send_message(QueueUrl=OUTQUEUE, - MessageBody=outfile) - - print('finished', body) - elif ((time.time() - LAST_MESSAGE_TIME) > - NOMSG_TIMEOUT_SECS): - print('No message received within timeout. Exiting') - break - else: - # sleep for a bit before checking again - time.sleep(30) - - -if __name__ == '__main__': - main() diff --git a/bin/rios_subproc_mpi.py b/bin/rios_subproc_mpi.py deleted file mode 100755 index 216e8e63..00000000 --- a/bin/rios_subproc_mpi.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -""" -Main program for RIOS MPI subprocesses. -This uses the MPI calls to receive and send data -from the main process. - -""" -# This file is part of RIOS - Raster I/O Simplification -# Copyright (C) 2012 Sam Gillingham, Neil Flood -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from __future__ import print_function - -from mpi4py import MPI -from rios.parallel import subproc - -from io import BytesIO - -if __name__ == "__main__": - - # get the handle to the parent - comm = MPI.Comm.Get_parent() - - # keep going until we are told to exit - while True: - # get the data from the parent - data = comm.recv(source=0) - - status, blockData = data - if not status: - comm.Disconnect() - break - - # wrap it with a BytesIO object so it can be treated - # like a file by subproc.runJob() - this keeps compatibility - # with the other sub job types - inf = BytesIO(blockData) - inf.seek(0) - - # output file object - outf = BytesIO() - - # do the processing - subproc.runJob(inf, outf) - - # send the result back - outdata = outf.getvalue() - comm.send(outdata, dest=0) - diff --git a/bin/rioscalcstats.py b/bin/rioscalcstats.py deleted file mode 100755 index 60fba89c..00000000 --- a/bin/rioscalcstats.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -""" -Use rios.calcstats to calculate statistics for the given image(s). - -""" - -from rios.cmdline import rioscalcstats -import warnings -warnings.warn("Future versions of RIOS may remove the .py extension from this script name", DeprecationWarning) - -rioscalcstats.main() diff --git a/bin/riosprintstats.py b/bin/riosprintstats.py deleted file mode 100755 index 86ad165f..00000000 --- a/bin/riosprintstats.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -""" -Use rios.fileinfo to print the statistics for the given image(s). - -""" - -from rios.cmdline import riosprintstats -import warnings -warnings.warn("Future versions of RIOS may remove the .py extension from this script name", DeprecationWarning) - -riosprintstats.main() diff --git a/bin/testrios.py b/bin/testrios.py deleted file mode 100755 index 28e90503..00000000 --- a/bin/testrios.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -""" -Main test harness for RIOS. - -Should be run as a main program. It then runs a selection -of tests of some capabilities of RIOS. - -""" -# This file is part of RIOS - Raster I/O Simplification -# Copyright (C) 2012 Sam Gillingham, Neil Flood -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys - -if __name__ == '__main__': - # the __name__ == '__main__' is required under Windows - # so they multiprocessing module works. - from rios.riostests import riostestutils - import warnings - warnings.warn("Future versions of RIOS may remove the .py extension from this script name", DeprecationWarning) - - if riostestutils.testAll() > 0: - # return error code - sys.exit(1) - else: - sys.exit(0) diff --git a/setup.py b/setup.py deleted file mode 100644 index f0c48210..00000000 --- a/setup.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -""" -The setup script for RIOS. Creates the module, installs -the scripts. -""" -# This file is part of RIOS - Raster I/O Simplification -# Copyright (C) 2012 Sam Gillingham, Neil Flood -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -from setuptools import setup -import glob -import sys -import site - - -site.ENABLE_USER_SITE = ("--user" in sys.argv[1:]) - -# When run via pyproject.toml, we seem unable to import our own package. -# To get around this, we need to add to the path. I have no idea why, -# but this seems to work fine. -sys.path.insert(0, '') -import rios # noqa: E402 - -# Are we installing the command line scripts? -# this is an experimental option for users who are -# using the Python entry point feature of setuptools and Conda instead -NO_INSTALL_CMDLINE = int(os.getenv('RIOS_NOCMDLINE', '0')) > 0 - -if NO_INSTALL_CMDLINE: - # still install the scripts for the parallel processing to work properly - scripts_list = ['bin/rios_subproc.py', 'bin/rios_subproc_mpi.py'] -else: - scripts_list = glob.glob("bin/*.py") - -setup(name='rios', - version=rios.RIOS_VERSION, - description='Raster Input/Output Simplification', - author='Sam Gillingham', - author_email='gillingham.sam@gmail.com', - scripts=scripts_list, - entry_points={ - 'console_scripts': [ - 'testrios = rios.riostests.riostestutils:testAll', - 'rioscalcstats = rios.cmdline.rioscalcstats:main', - 'riosprintstats = rios.cmdline.riosprintstats:main', - 'rios_computeworker = rios.cmdline.rios_computeworker:mainCmd' - ]}, - packages=['rios', 'rios/parallel', 'rios/parallel/aws', - 'rios/riostests', 'rios/cmdline'], - license='LICENSE.txt', - url='https://www.rioshome.org' - )