first commit

bin/wreduce-folder

@@ -0,0 +1,81 @@
+import os
+import pyfits
+import watcherreduce as wr
+from watcherreduce import reduce
+import sys, optparse
+
+import datetime
+
+class flatcache:
+	def __init__(self):
+		self.cache = {}
+
+	def get(self, flatdate, filter):
+		return self.cache.get(str(flatdate.date()), {}).get(filter)
+
+	def save(self, flatdate, filter, hdu):
+		nightdict = self.cache.get(str(flatdate.date()))
+		if not nightdict: self.cache[str(flatdate.date())] = {}
+		self.cache[str(flatdate.date())][filter] = hdu
+
+
+def easyReduceFolder(inputfolder, outputfolder, suffix='_red'):
+	files = [os.path.join(inputfolder, filename) for filename in os.listdir(inputfolder) if filename.endswith('.fits')]
+	files.sort()
+	downloaded = flatcache()
+	for fitspath in files:
+		imagehdu = pyfits.open(fitspath)[0]
+		flatNight, filter = wr.findFlatDetailsForHDU(imagehdu)
+		# Do we already have a master flat that matches this night? 
+		masterflathdu = downloaded.get(flatNight, filter)
+		if not masterflathdu:
+			# If we don't, what is the correct flat for this night?
+			validNight = wr.searchMasterFlat(flatNight, filter, limit = 100, allowNewer = True)
+			if not validNight:
+				# If we can't find a valid flat, just skip reducing this file.
+				print 'Unable to find valid flat for %s, skipping.' % os.path.split(fitspath)[1]
+				continue
+			# Did we already download it?
+			masterflathdu = downloaded.get(validNight, filter)
+			if not masterflathdu:
+			# If not, let's download it.
+				masterflathdu = wr.downloadMasterFlat(validNight, filter)
+				# And let's store it.
+				downloaded.save(validNight, filter, masterflathdu)
+			# Update the cache to point to it in future.
+			downloaded.save(flatNight, filter, masterflathdu)
+
+
+		reducedhdu = reduce.reduceImage(imagehdu, masterflathdu)
+		filename, extension = os.path.splitext(os.path.split(fitspath)[1])
+		outputname = filename + suffix + extension
+		outputpath = os.path.join(outputfolder, outputname)
+		reducedhdu.writeto(outputpath)
+
+
+def main(argv):
+	parser = optparse.OptionParser(usage = '%prog <input folder> [options]')
+	parser.add_option('-o', '--output', help = 'Specify an output folder')
+	parser.add_option('-s', '--suffix', help = 'Specify a suffix to be appended to processed filenames before the extension', default = '_red')
+
+	(opts, args) = parser.parse_args()
+
+	if len(args) < 1:
+		print "Need to specify input folder. Use '.' for current folder."
+		return
+	else:
+		inputfolder = args[0]
+
+	outputfolder = opts.output
+	if outputfolder:
+		if not os.path.exists(outputfolder):
+			print 'Output folder must already exist.'
+			return
+	else:
+		outputfolder = ''
+
+	easyReduceFolder(inputfolder, outputfolder, suffix = opts.suffix)
+
+
+if __name__ == '__main__':
+	main(sys.argv)

setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup
+
+setup(	name='watcherreduce',
+		version='0.1',
+		description='Tools for interacting with the Watcher Data Repo.',
+		author='David Murphy',
+		author_email='[email protected]',
+		packages=['watcherreduce'],
+		install_requires=['astropy','numpy'],
+		scripts=['bin/wreduce-folder'],
+		zip_safe=False
+		)

watcherreduce.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: watcherreduce
+Version: 0.1
+Summary: Tools for interacting with the Watcher Data Repo.
+Home-page: UNKNOWN
+Author: David Murphy
+Author-email: [email protected]
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

watcherreduce.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+setup.py
+bin/wreduce-folder
+watcherreduce/__init__.py
+watcherreduce/reduce.py
+watcherreduce.egg-info/PKG-INFO
+watcherreduce.egg-info/SOURCES.txt
+watcherreduce.egg-info/dependency_links.txt
+watcherreduce.egg-info/not-zip-safe
+watcherreduce.egg-info/requires.txt
+watcherreduce.egg-info/top_level.txt

watcherreduce.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

watcherreduce.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

watcherreduce.egg-info/requires.txt

@@ -0,0 +1,2 @@
+astropy
+numpy

watcherreduce.egg-info/top_level.txt

@@ -0,0 +1 @@
+watcherreduce

watcherreduce/__init__.py

@@ -0,0 +1,83 @@
+# Work with Watcher Data
+# Functions to load Watcher FITS files and automatically reduce them
+
+import astropy.io.fits as pyfits
+from watcherreduce import reduce
+import dateutil.parser
+import os
+import datetime
+
+import requests
+import io
+
+watcherserver = 'http://ssamr.ucd.ie'
+
+from pynapple import getpynfilter, HEADERCONFIG
+
+
+def searchMasterFlat(night, filter, limit=100, allowNewer=True):
+	url = "%s/flatsearch?date=%04d-%02d-%02d&filter=%s&newer=%s&limit=%d" % (watcherserver, night.year, night.month, night.day, filter, allowNewer, limit)
+	response = requests.get(url)
+	if response.status_code == requests.codes.ok:
+		datestring = response.json()['date']
+		flatdate = dateutil.parser.parse(datestring)
+		return flatdate
+	else:
+		return None
+
+def downloadMasterFlat(night, filter):
+	url = "%s/getflat?date=%04d-%02d-%02d&filter=%s" % (watcherserver, night.year, night.month, night.day, filter)
+	response = requests.get(url)
+	fitsfile = io.BytesIO(response.content)
+	masterflathdu = pyfits.open(fitsfile)[0]
+	return masterflathdu
+
+
+def thisWatcherNight():
+	now = datetime.datetime.utcnow()
+	return watcherNight(now)
+
+
+def watcherNight(imageDateTime):
+	day = imageDateTime - datetime.timedelta(days = (imageDateTime.hour < 12))
+#	date = day.date()
+	return day
+
+
+# Given a path to a FITS file, finds the Watcher night and filter. 
+def findFlatDetailsForImage(fitspath,  headerconfig=HEADERCONFIG):
+	imagehdu = pyfits.open(fitspath)[0]
+	return findFlatDetailsForHDU(imagehdu,  headerconfig=headerconfig)
+
+
+# Given an image HDU, finds the Watcher night and filter. 
+def findFlatDetailsForHDU(imagehdu,  headerconfig=HEADERCONFIG):
+	filter = getpynfilter(imagehdu.header)
+	headerTime = imagehdu.header[headerconfig.get('Basic','datetime')]
+	imageNight = watcherNight(dateutil.parser.parse(headerTime))
+	return imageNight, filter
+
+
+def loadReducedFits(fitspath, flatpath=None, biaspath=None, flatNight=None, headerconfig=HEADERCONFIG):
+	imagehdu = pyfits.open(fitspath)[0]
+	filter = getpynfilter(imagehdu.header)
+
+	if flatpath:
+		masterflathdu = pyfits.open(flatpath)[0]
+	else:
+		if not flatNight:
+			flatNight, __ = findFlatDetailsForHDU(imagehdu, headerconfig=headerconfig)
+			validNight = searchMasterFlat(flatNight, filter, limit = 100, allowNewer = True)		
+		masterflathdu = downloadMasterFlat(validNight, filter)
+	if biaspath:
+		biashdu = pyfits.open(biaspath)[0]
+	else:
+		biashdu = None	
+	reducedhdu = reduce.reduceImage(imagehdu, masterflathdu, bias=biashdu)
+	return reducedhdu
+
+
+
+
+
+

watcherreduce/reduce.py

@@ -0,0 +1,21 @@
+import copy
+import numpy as np
+
+
+def reduceImage(imagehdu, masterflathdu, bias=None):
+	biasdata = safeBiasData(bias)
+	reduceddata = (imagehdu.data - biasdata)/masterflathdu.data
+	reducedhdu = copy.deepcopy(imagehdu)
+	reducedhdu.data[:,:] = reduceddata[:,:]
+	return reducedhdu
+
+
+def safeBiasData(biashdu):
+	if biashdu:
+		biasdata = biashdu.data
+	else:
+		biasdata = np.full((1024,1024), 400) # Maybe fix hardcoded size in future!
+	return biasdata
+
+
+