Implement reduction for DIPOL instrument

.github/workflows/ci.yml

@@ -75,15 +75,18 @@ jobs:
       - name: Download test data
         env:
           TEST_DATA_PASSWORD: ${{ secrets.test_data_password }}
-        run:  wget --post-data "pass=$TEST_DATA_PASSWORD" 'https://vhega.iaa.es/iop4/iop4testdata.tar.gz' -O $HOME/iop4testdata.tar.gz
+        run:  | 
+          export TESTDATA_MD5SUM=`grep 'TESTDATA_MD5SUM' ./tests/conftest.py | awk -F"'" '{print $2}' | tr -d '\n'`
+          wget --post-data "pass=$TEST_DATA_PASSWORD" "https://vhega.iaa.es/iop4/iop4testdata.tar.gz?md5sum=$TESTDATA_MD5SUM" -O $HOME/iop4testdata.tar.gz
 
       - name: Run tests (with -o log_cli=true -o log_cli_level=DEBUG to debug CI actions)
-        run: pytest -o log_cli=true -o log_cli_level=DEBUG -vxs tests
+        run: pytest -o log_cli=true -o log_cli_level=DEBUG -vxs tests/
 
-      - name: Output some info for debugging
+      - name: Output some info for debugging 
+        # | true so erros in this step are ignored
         run: |
-          du -sh $HOME/.cache/httpdirfs/
-
+          df -h || true
+          du -sh $HOME/.cache/httpdirfs/ || true
 
 
 

README.md

@@ -56,7 +56,7 @@ To run the tests, first follow the previous steps to configure IOP4. At the mome
 ```
 If it is the first time executing IOP4, the astrometry index files will be downloaded to `astrometry_cache_path` (see `config/config.example.yaml`). This will take some time and a few tens of GB, depending on the exact version.
 
-**Warning**: in some macOS systems, the process [might hang up](https://github.com/juanep97/iop4/issues/14#issuecomment-1748465276). Execute `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` or add that line to you shell init script.
+**Warning**: in some macOS systems, the process [might hang up](https://github.com/juanep97/iop4/issues/14#issuecomment-1748465276). Execute `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` or add that line to your shell init script.
 
 ## Usage
 

config/config.example.yaml

@@ -1,43 +1,75 @@
 # This is an example of the config file, copy it to config/config.yaml and edit it.
 # IOP4LIB will attempt to use config/config.yaml if it exists, otherwise it will use this one.
 
-# GENERAL
+###############
+### GENERAL ###
+###############
+
 datadir: ~/.iop4data/ # <str> Path to iop4data data folder.
 set_rawdata_readonly: False # <bool> True / False (sets raw fits file to readonly when downloading them or creating a RawFit object).
 db_path: ~/.iop4data/iop4.db # Path to iop4 sqlite database file.
 astrometry_cache_path: ~/.astrometry_cache/ # <tr> Path to store the astromery index files.
 max_concurrent_threads: 4 # <int> Number of threads / processes to use (e.g. 4).
+astrometry_timeout: 20 # <int> Timeout in minutes for astrometry solving.
+
+###################
+### RAY CLUSTER ###
+###################
 
-# RAY CLUSTER
 ray_use_cluster: False # <bool> True/False (use ray for parallelization), let it to false if you have not configured it.
 ray_cluster_address: null # <str> Aaddress for the cluster, needs ssh keys for current user; e.g. 'user@address'.
 ray_cluster_config: null # <str> Path to ray cluster config file, e.g. /path/to/iop4/priv.rayconfig.yaml'.
 ray_db_path: null # <str> Path in ray cluster, e.g. '~/iop4data/iop4.db'.
 ray_datadir: null # <str> Path in ray cluster, e.g. '~/iop4data/'.
 
-# GRAPHICS
+################
+### GRAPHICS ###
+################
+
 mplt_default_dpi: 100 # <int> dpi for matplotlib (e.g. 100)
 
-# LOGGING
+###############
+### LOGGING ###
+###############
+
 log_fname: ~/.iop4data/iop4.log # </path/to/iop4.log>
 log_date_format: '%Y-%m-%d %H:%M:%S' # <log date format>
-log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # <log format>
+log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # <log format> You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process.
 log_level: 20 # <int>. Possible values are: 10 (DEBUG), 20 (INFO), 30 (WARNING), 40 (ERROR), 50 (CRITICAL).
 
-# CAHA
+############
+### CAHA ###
+############
+
 caha_address: null # <caha ftp address>
 caha_password: null # <your caha password>
 caha_user: null # <your caha user>
 
-# OSN
+###########
+### OSN ###
+###########
+
 osn_t090_address: null # <osn t090 ftp archive ip address>
 osn_t090_user: null # <yourosnuser>
 osn_t090_password: null # <yourosnpassword>
+
 osn_t150_address: null # <osn t150 ftp archive ip address>
 osn_t150_user: null # <yourosnuser>
 osn_t150_password: null # <yourosnpassword>
+
+# List of names to download from OSN.
+# The names will be introduced in osn_fnames_patterns as 'name.*\.fi?ts?$' (see below).
+
 osn_source_list_path: null # <path/to/osn/source/list>
-osn_fnames_patterns: # List of regex patterns. Files matching any of these patterns will be downloaded from OSN. For example, (^BLLac.*\.fits?$) will match file names starting with BLLac and ending in .fit or .fits. You can include your initials here, if you want to download only your files and the observers include it in the file name, e.g. (.*mfz.*\.fits?$) for files containing mfz in the name could belong to Menganito Fulano Zutano.
-- (^Flat.*\.fits?$)
-- (^Bias.*\.fits?$)
-# osn_download_all_then_check_owner: null # <str> Alternatively, for OSN, just download all files (^.*\.fits?$) and then remove those that contain this word in the OBSERVER keyword in the header (except for files that contain 'bias' or 'flat' in the name). Leave null for default behaviour.
+
+# List of (case insentitive) regex patterns.
+# Files matching any of these patterns will be downloaded from OSN. 
+# For example, (^BLLac.*\.fits?$) will match file names starting with BLLac 
+# and ending in .fit, .fits, or .fts. You can include your initials here, 
+# If you want to download only your files and the observers include it in the 
+# filename, e.g. (.*mfz.*\.fits?$) for files containing mfz in the name 
+# could belong to Menganito Fulano Zutano. The re.IGNORECASE flag will be passed.
+
+osn_fnames_patterns: 
+- (^Flat.*\.fi?ts?$)
+- (^Bias.*\.fi?ts?$)

docs/citations.bib

@@ -45,3 +45,7 @@ @article{dipol:2020
 	pages = "A46",
 }
 
+@online{astropy:ccd_data_reduction_guide,
+      title        = {CCD Data Reduction Guide},
+      url          = {https://www.astropy.org/ccd-reduction-and-photometry-guide/},
+}

iop4admin/modeladmins/aperphotresult.py

@@ -17,17 +17,21 @@
 
 class AdminAperPhotResult(admin.ModelAdmin):
     model = AperPhotResult
-    list_display = ['id', 'get_telescope', 'get_datetime', 'get_src_name', 'get_src_type', 'get_fwhm', 'get_aperpix', 'get_reducedfit', 'get_obsmode', 'pairs', 'get_rotangle', 'get_src_type', 'get_flux_counts', 'get_flux_counts_err', 'get_bkg_flux_counts', 'get_bkg_flux_counts_err', 'modified']
+    list_display = ['id', 'get_telescope', 'get_instrument', 'get_datetime', 'get_src_name', 'get_src_type', 'get_fwhm', 'get_aperpix', 'get_reducedfit', 'get_obsmode', 'pairs', 'get_rotangle', 'get_src_type', 'get_flux_counts', 'get_flux_counts_err', 'get_bkg_flux_counts', 'get_bkg_flux_counts_err', 'modified']
     readonly_fields = [field.name for field in AperPhotResult._meta.fields]
-    search_fields = ['id', 'astrosource__name', 'astrosource__srctype', 'reducedfit__id']
-    list_filter = ['astrosource__srctype', 'reducedfit__epoch__telescope', 'reducedfit__obsmode']
+    search_fields = ['id', 'reducedfit__instrument', 'astrosource__name', 'astrosource__srctype', 'reducedfit__id']
+    list_filter = ['reducedfit__instrument', 'astrosource__srctype', 'reducedfit__epoch__telescope', 'reducedfit__obsmode']
 
 
 
     @admin.display(description="TELESCOPE")
     def get_telescope(self, obj):
         return obj.reducedfit.epoch.telescope
 
+    @admin.display(description="INSTRUMENT")
+    def get_instrument(self, obj):
+        return obj.reducedfit.instrument
+
     @admin.display(description="DATETIME")
     def get_datetime(self, obj):
         return Time(obj.reducedfit.juliandate, format='jd').strftime('%Y-%m-%d %H:%M:%S')

iop4admin/modeladmins/epoch.py

@@ -17,6 +17,7 @@ class AdminEpoch(admin.ModelAdmin):
     search_fields = ['id', 'telescope', 'night']
     readonly_fields = [field.name for field in Epoch._meta.fields] 
     ordering = ['-night','-telescope']
+    list_filter = ['telescope', 'night']
 
 
 

iop4admin/modeladmins/fitfile.py

@@ -24,6 +24,22 @@
 
 class AdminFitFile(admin.ModelAdmin):
 
+    @admin.display(description='TELESCOPE', ordering='epoch__telescope')
+    def telescope(self, obj):
+        return obj.epoch.telescope
+
+    @admin.display(description='NIGHT', ordering='epoch__night')
+    def night(self, obj):
+        return obj.epoch.night
+
+    @admin.display(description='FILENAME', ordering='filename')
+    def filename(self, obj):
+        return obj.filename
+
+    @admin.display(description='STATUS')
+    def status(self, obj):
+        return ", ".join(obj.flag_labels)
+
     def get_urls(self):
         urls = super().get_urls()
         my_urls = [

iop4admin/modeladmins/masterbias.py

@@ -11,10 +11,20 @@
 logger = logging.getLogger(__name__)
 
 class AdminMasterBias(AdminFitFile):
+
     model = MasterBias
-    list_display = ['id', 'telescope', 'night', 'imgsize', 'get_built_from', 'options']
 
+    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'get_built_from', 'options']
 
+    list_filter = (
+            RawFitIdFilter,
+            RawFitTelescopeFilter,
+            RawFitNightFilter,
+            RawFitInstrumentFilter,
+            RawFitFlagFilter,
+            "imgsize",
+    )
+
 
     @admin.display(description='Options')
     def options(self, obj):

iop4admin/modeladmins/masterdark.py

@@ -0,0 +1,58 @@
+from django.contrib import admin
+
+from django.utils.html import format_html
+from django.urls import reverse 
+from django.utils.safestring import mark_safe
+
+from iop4api.filters import *
+from iop4api.models import *
+from .fitfile import AdminFitFile
+
+import logging
+logger = logging.getLogger(__name__)
+
+class AdminMasterDark(AdminFitFile):
+
+    model = MasterDark
+
+    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'exptime', 'get_masterbias', 'get_built_from', 'options']
+
+    list_filter = (
+            RawFitIdFilter,
+            RawFitTelescopeFilter,
+            RawFitNightFilter,
+            RawFitInstrumentFilter,
+            RawFitFlagFilter,
+            "imgsize",
+    )
+
+    @admin.display(description='Options')
+    def options(self, obj):
+        url_details = reverse('iop4admin:iop4api_masterdark_details', args=[obj.id])
+        url_viewer= reverse('iop4admin:iop4api_masterdark_details', args=[obj.id])
+        return format_html(rf'<a href="{url_details}">details</a> / <a href="{url_viewer}">advanced viewer</a>')
+
+    @admin.display(description='Telescope')
+    def telescope(self, obj):
+        return obj.epoch.telescope
+
+    @admin.display(description='Night')
+    def night(self, obj):
+        return obj.epoch.night
+
+    @admin.display(description='MasterBias')
+    def get_masterbias(self, obj):
+        self.allow_tags = True
+        if obj.masterbias is None:
+            return "-"
+        url = reverse('iop4admin:%s_%s_changelist' % (MasterBias._meta.app_label, MasterBias._meta.model_name)) + f"?id={obj.masterbias.id}"
+        return mark_safe(rf'<a href="{url}">{obj.masterbias.id}</a>')
+
+    @admin.display(description="Built from")
+    def get_built_from(self, obj):
+        self.allow_tags = True
+        link_L = list()
+        for rawfit in obj.rawfits.all():
+            url = reverse('iop4admin:%s_%s_changelist' % (RawFit._meta.app_label, RawFit._meta.model_name)) + f"?id={rawfit.id}"
+            link_L.append(rf'<a href="{url}">{rawfit.id}</a>')
+        return mark_safe(", ".join(link_L))

iop4admin/modeladmins/masterflat.py

@@ -12,9 +12,19 @@
 logger = logging.getLogger(__name__)
 
 class AdminMasterFlat(AdminFitFile):
+
     model = MasterFlat
-    list_display = ['id', 'telescope', 'night', 'imgsize', 'band', 'obsmode', 'rotangle', 'exptime', 'masterbias', 'get_built_from', 'options']
 
+    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'band', 'obsmode', 'rotangle', 'exptime', 'get_masterbias', 'get_built_from', 'options']
+
+    list_filter = (
+            RawFitIdFilter,
+            RawFitTelescopeFilter,
+            RawFitNightFilter,
+            RawFitInstrumentFilter,
+            RawFitFlagFilter,
+            "imgsize",
+    )
 
 
     @admin.display(description='Options')
@@ -31,6 +41,14 @@ def telescope(self, obj):
     def night(self, obj):
         return obj.epoch.night
 
+    @admin.display(description='MasterBias')
+    def get_masterbias(self, obj):
+        self.allow_tags = True
+        if obj.masterbias is None:
+            return "-"
+        url = reverse('iop4admin:%s_%s_changelist' % (MasterBias._meta.app_label, MasterBias._meta.model_name)) + f"?id={obj.masterbias.id}"
+        return mark_safe(rf'<a href="{url}">{obj.masterbias.id}</a>')
+
     @admin.display(description="Built from")
     def get_built_from(self, obj):
         self.allow_tags = True

iop4admin/modeladmins/photopolresult.py

@@ -15,7 +15,7 @@
 
 class AdminPhotoPolResult(admin.ModelAdmin):
     model = PhotoPolResult
-    list_display = ['id', 'get_telescope', 'get_juliandate', 'get_datetime', 'get_src_name', 'get_src_type', 'get_reducedfits', 'obsmode', 'band', 'exptime', 'get_mag', 'get_mag_err', 'get_p', 'get_p_err', 'get_chi', 'get_chi_err', 'modified']
+    list_display = ['id', 'get_telescope', 'get_juliandate', 'get_datetime', 'get_src_name', 'get_src_type', 'get_reducedfits', 'obsmode', 'band', 'exptime', 'get_mag', 'get_mag_err', 'get_p', 'get_p_err', 'get_chi', 'get_chi_err', 'get_flags', 'modified']
     readonly_fields = [field.name for field in PhotoPolResult._meta.fields]
     search_fields = ['id', 'astrosource__name', 'astrosource__srctype', 'epoch__night']
     ordering = ['-juliandate']
@@ -42,11 +42,11 @@ def get_src_type(self, obj):
     @admin.display(description="ReducedFits")
     def get_reducedfits(self, obj):
         self.allow_tags = True
-        link_L = list()
-        for reducedfit in obj.reducedfits.all():
-            url = reverse('iop4admin:%s_%s_changelist' % (ReducedFit._meta.app_label, ReducedFit._meta.model_name)) + f"?id={reducedfit.id}"
-            link_L.append(rf'<a href="{url}">{reducedfit.id}</a>')
-        return mark_safe(", ".join(link_L))
+
+        ids_str_L = [str(reducedfit.id) for reducedfit in obj.reducedfits.all()]
+        a_href = reverse('iop4admin:%s_%s_changelist' % (ReducedFit._meta.app_label, ReducedFit._meta.model_name)) + "?id__in=%s" % ",".join(ids_str_L)
+        a_text = ", ".join(ids_str_L)
+        return mark_safe(f'<a href="{a_href}">{a_text}</a>')
 
     @admin.display(description="JD")
     def get_juliandate(self, obj):
@@ -74,4 +74,8 @@ def get_chi(self, obj):
 
     @admin.display(description="CHIERR [º]", ordering='-chi_err')
     def get_chi_err(self, obj):
-        return f"{obj.chi_err:.2f}" if obj.chi_err is not None else None
+        return f"{obj.chi_err:.2f}" if obj.chi_err is not None else None
+
+    @admin.display(description='Status')
+    def get_flags(self, obj):
+        return ", ".join(list(obj.flag_labels))

iop4admin/modeladmins/rawfit.py

@@ -14,7 +14,7 @@
 
 class AdminRawFit(AdminFitFile):
     model = RawFit
-    list_display = ["id", 'filename', 'telescope', 'night', 'status', 'imgtype', 'imgsize', 'band', 'obsmode', 'rotangle', 'exptime', 'options']
+    list_display = ["id", 'filename', 'telescope', 'night', 'instrument', 'status', 'imgtype', 'imgsize', 'band', 'obsmode', 'rotangle', 'exptime', 'options']
     readonly_fields = [field.name for field in RawFit._meta.fields] 
     search_fields = ['id', 'filename', 'epoch__telescope', 'epoch__night'] 
     ordering = ['-epoch__night','-epoch__telescope']
@@ -24,36 +24,28 @@ class AdminRawFit(AdminFitFile):
             RawFitTelescopeFilter,
             RawFitNightFilter,
             RawFitFilenameFilter,
+            RawFitInstrumentFilter,
             RawFitFlagFilter,
             "imgtype",
             "obsmode",
             "band",
             "imgsize",
         )
 
-
-
-    def telescope(self, obj):
-        return obj.epoch.telescope
-
-    def night(self, obj):
-        return obj.epoch.night
-
-    @admin.display(description='STATUS')
-    def status(self, obj):
-        return ", ".join(obj.flag_labels)
 
     @admin.display(description='OPTIONS')
     def options(self, obj):
-        if obj.imgtype == IMGTYPES.LIGHT:
+        html_src = str()
+
+        if obj.imgtype == IMGTYPES.LIGHT and hasattr(obj, "reduced"):
             url_reduced = reverse('iop4admin:%s_%s_changelist' % (ReducedFit._meta.app_label, ReducedFit._meta.model_name)) + f"?id={obj.reduced.id}"
-            url_details = reverse('iop4admin:iop4api_rawfit_details', args=[obj.id])
-            url_viewer= reverse('iop4admin:iop4api_rawfit_viewer', args=[obj.id])
-            return format_html(rf'<a href="{url_reduced}">reduced</a> / <a href="{url_details}">details</a> / <a href="{url_viewer}">advanced viewer</a>')
-        else:
-            url_details = reverse('iop4admin:iop4api_rawfit_details', args=[obj.id])
-            url_viewer= reverse('iop4admin:iop4api_rawfit_viewer', args=[obj.id])
-            return format_html(rf'<a href="{url_details}">details</a> / <a href="{url_viewer}">advanced viewer</a>')
+            html_src += rf'<a href="{url_reduced}">reduced</a> / '
+
+        url_details = reverse('iop4admin:iop4api_rawfit_details', args=[obj.id])
+        url_viewer= reverse('iop4admin:iop4api_rawfit_viewer', args=[obj.id])
+        html_src += rf'<a href="{url_details}">details</a> / <a href="{url_viewer}">advanced viewer</a>'
+
+        return format_html(html_src)
 
     def image_preview(self, obj, allow_tags=True):
         url_img_preview = reverse('iop4admin:iop4api_rawfit_preview', args=[obj.id])