From 41076e7e479414ba6efd8f3621c234199842704c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 25 May 2016 12:41:38 +0100 Subject: [PATCH] first commit --- bin/wreduce-folder | 81 ++++++++++++++++++++ setup.py | 12 +++ watcherreduce.egg-info/PKG-INFO | 10 +++ watcherreduce.egg-info/SOURCES.txt | 10 +++ watcherreduce.egg-info/dependency_links.txt | 1 + watcherreduce.egg-info/not-zip-safe | 1 + watcherreduce.egg-info/requires.txt | 2 + watcherreduce.egg-info/top_level.txt | 1 + watcherreduce/__init__.py | 83 +++++++++++++++++++++ watcherreduce/reduce.py | 21 ++++++ 10 files changed, 222 insertions(+) create mode 100644 bin/wreduce-folder create mode 100644 setup.py create mode 100644 watcherreduce.egg-info/PKG-INFO create mode 100644 watcherreduce.egg-info/SOURCES.txt create mode 100644 watcherreduce.egg-info/dependency_links.txt create mode 100644 watcherreduce.egg-info/not-zip-safe create mode 100644 watcherreduce.egg-info/requires.txt create mode 100644 watcherreduce.egg-info/top_level.txt create mode 100644 watcherreduce/__init__.py create mode 100644 watcherreduce/reduce.py diff --git a/bin/wreduce-folder b/bin/wreduce-folder new file mode 100644 index 0000000..3b35039 --- /dev/null +++ b/bin/wreduce-folder @@ -0,0 +1,81 @@ +import os +import pyfits +import watcherreduce as wr +from watcherreduce import reduce +import sys, optparse + +import datetime + +class flatcache: + def __init__(self): + self.cache = {} + + def get(self, flatdate, filter): + return self.cache.get(str(flatdate.date()), {}).get(filter) + + def save(self, flatdate, filter, hdu): + nightdict = self.cache.get(str(flatdate.date())) + if not nightdict: self.cache[str(flatdate.date())] = {} + self.cache[str(flatdate.date())][filter] = hdu + + +def easyReduceFolder(inputfolder, outputfolder, suffix='_red'): + files = [os.path.join(inputfolder, filename) for filename in os.listdir(inputfolder) if filename.endswith('.fits')] + files.sort() + downloaded = flatcache() + for fitspath in files: + imagehdu = pyfits.open(fitspath)[0] + flatNight, filter = wr.findFlatDetailsForHDU(imagehdu) + # Do we already have a master flat that matches this night? + masterflathdu = downloaded.get(flatNight, filter) + if not masterflathdu: + # If we don't, what is the correct flat for this night? + validNight = wr.searchMasterFlat(flatNight, filter, limit = 100, allowNewer = True) + if not validNight: + # If we can't find a valid flat, just skip reducing this file. + print 'Unable to find valid flat for %s, skipping.' % os.path.split(fitspath)[1] + continue + # Did we already download it? + masterflathdu = downloaded.get(validNight, filter) + if not masterflathdu: + # If not, let's download it. + masterflathdu = wr.downloadMasterFlat(validNight, filter) + # And let's store it. + downloaded.save(validNight, filter, masterflathdu) + # Update the cache to point to it in future. + downloaded.save(flatNight, filter, masterflathdu) + + + reducedhdu = reduce.reduceImage(imagehdu, masterflathdu) + filename, extension = os.path.splitext(os.path.split(fitspath)[1]) + outputname = filename + suffix + extension + outputpath = os.path.join(outputfolder, outputname) + reducedhdu.writeto(outputpath) + + +def main(argv): + parser = optparse.OptionParser(usage = '%prog [options]') + parser.add_option('-o', '--output', help = 'Specify an output folder') + parser.add_option('-s', '--suffix', help = 'Specify a suffix to be appended to processed filenames before the extension', default = '_red') + + (opts, args) = parser.parse_args() + + if len(args) < 1: + print "Need to specify input folder. Use '.' for current folder." + return + else: + inputfolder = args[0] + + outputfolder = opts.output + if outputfolder: + if not os.path.exists(outputfolder): + print 'Output folder must already exist.' + return + else: + outputfolder = '' + + easyReduceFolder(inputfolder, outputfolder, suffix = opts.suffix) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6d2ba6f --- /dev/null +++ b/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +setup( name='watcherreduce', + version='0.1', + description='Tools for interacting with the Watcher Data Repo.', + author='David Murphy', + author_email='david@spacescience.ie', + packages=['watcherreduce'], + install_requires=['astropy','numpy'], + scripts=['bin/wreduce-folder'], + zip_safe=False + ) diff --git a/watcherreduce.egg-info/PKG-INFO b/watcherreduce.egg-info/PKG-INFO new file mode 100644 index 0000000..04aff78 --- /dev/null +++ b/watcherreduce.egg-info/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: watcherreduce +Version: 0.1 +Summary: Tools for interacting with the Watcher Data Repo. +Home-page: UNKNOWN +Author: David Murphy +Author-email: david@spacescience.ie +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN diff --git a/watcherreduce.egg-info/SOURCES.txt b/watcherreduce.egg-info/SOURCES.txt new file mode 100644 index 0000000..309f713 --- /dev/null +++ b/watcherreduce.egg-info/SOURCES.txt @@ -0,0 +1,10 @@ +setup.py +bin/wreduce-folder +watcherreduce/__init__.py +watcherreduce/reduce.py +watcherreduce.egg-info/PKG-INFO +watcherreduce.egg-info/SOURCES.txt +watcherreduce.egg-info/dependency_links.txt +watcherreduce.egg-info/not-zip-safe +watcherreduce.egg-info/requires.txt +watcherreduce.egg-info/top_level.txt \ No newline at end of file diff --git a/watcherreduce.egg-info/dependency_links.txt b/watcherreduce.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/watcherreduce.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/watcherreduce.egg-info/not-zip-safe b/watcherreduce.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/watcherreduce.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/watcherreduce.egg-info/requires.txt b/watcherreduce.egg-info/requires.txt new file mode 100644 index 0000000..8f99f06 --- /dev/null +++ b/watcherreduce.egg-info/requires.txt @@ -0,0 +1,2 @@ +astropy +numpy \ No newline at end of file diff --git a/watcherreduce.egg-info/top_level.txt b/watcherreduce.egg-info/top_level.txt new file mode 100644 index 0000000..e4060f4 --- /dev/null +++ b/watcherreduce.egg-info/top_level.txt @@ -0,0 +1 @@ +watcherreduce diff --git a/watcherreduce/__init__.py b/watcherreduce/__init__.py new file mode 100644 index 0000000..766403c --- /dev/null +++ b/watcherreduce/__init__.py @@ -0,0 +1,83 @@ +# Work with Watcher Data +# Functions to load Watcher FITS files and automatically reduce them + +import astropy.io.fits as pyfits +from watcherreduce import reduce +import dateutil.parser +import os +import datetime + +import requests +import io + +watcherserver = 'http://ssamr.ucd.ie' + +from pynapple import getpynfilter, HEADERCONFIG + + +def searchMasterFlat(night, filter, limit=100, allowNewer=True): + url = "%s/flatsearch?date=%04d-%02d-%02d&filter=%s&newer=%s&limit=%d" % (watcherserver, night.year, night.month, night.day, filter, allowNewer, limit) + response = requests.get(url) + if response.status_code == requests.codes.ok: + datestring = response.json()['date'] + flatdate = dateutil.parser.parse(datestring) + return flatdate + else: + return None + +def downloadMasterFlat(night, filter): + url = "%s/getflat?date=%04d-%02d-%02d&filter=%s" % (watcherserver, night.year, night.month, night.day, filter) + response = requests.get(url) + fitsfile = io.BytesIO(response.content) + masterflathdu = pyfits.open(fitsfile)[0] + return masterflathdu + + +def thisWatcherNight(): + now = datetime.datetime.utcnow() + return watcherNight(now) + + +def watcherNight(imageDateTime): + day = imageDateTime - datetime.timedelta(days = (imageDateTime.hour < 12)) +# date = day.date() + return day + + +# Given a path to a FITS file, finds the Watcher night and filter. +def findFlatDetailsForImage(fitspath, headerconfig=HEADERCONFIG): + imagehdu = pyfits.open(fitspath)[0] + return findFlatDetailsForHDU(imagehdu, headerconfig=headerconfig) + + +# Given an image HDU, finds the Watcher night and filter. +def findFlatDetailsForHDU(imagehdu, headerconfig=HEADERCONFIG): + filter = getpynfilter(imagehdu.header) + headerTime = imagehdu.header[headerconfig.get('Basic','datetime')] + imageNight = watcherNight(dateutil.parser.parse(headerTime)) + return imageNight, filter + + +def loadReducedFits(fitspath, flatpath=None, biaspath=None, flatNight=None, headerconfig=HEADERCONFIG): + imagehdu = pyfits.open(fitspath)[0] + filter = getpynfilter(imagehdu.header) + + if flatpath: + masterflathdu = pyfits.open(flatpath)[0] + else: + if not flatNight: + flatNight, __ = findFlatDetailsForHDU(imagehdu, headerconfig=headerconfig) + validNight = searchMasterFlat(flatNight, filter, limit = 100, allowNewer = True) + masterflathdu = downloadMasterFlat(validNight, filter) + if biaspath: + biashdu = pyfits.open(biaspath)[0] + else: + biashdu = None + reducedhdu = reduce.reduceImage(imagehdu, masterflathdu, bias=biashdu) + return reducedhdu + + + + + + diff --git a/watcherreduce/reduce.py b/watcherreduce/reduce.py new file mode 100644 index 0000000..0b6fb55 --- /dev/null +++ b/watcherreduce/reduce.py @@ -0,0 +1,21 @@ +import copy +import numpy as np + + +def reduceImage(imagehdu, masterflathdu, bias=None): + biasdata = safeBiasData(bias) + reduceddata = (imagehdu.data - biasdata)/masterflathdu.data + reducedhdu = copy.deepcopy(imagehdu) + reducedhdu.data[:,:] = reduceddata[:,:] + return reducedhdu + + +def safeBiasData(biashdu): + if biashdu: + biasdata = biashdu.data + else: + biasdata = np.full((1024,1024), 400) # Maybe fix hardcoded size in future! + return biasdata + + +