From 2bab0927a57bbac43d64da9f8c4fd13c62dabeae Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 14:48:58 -0300 Subject: [PATCH 01/15] added initial idea with tests --- alerce/search.py | 20 ++++++++++++++++++++ tests/test_search.py | 23 +++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/alerce/search.py b/alerce/search.py index 9185089..571c764 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -159,6 +159,26 @@ def query_non_detections(self, oid, format="json", index=None, sort=None): "GET", self.__get_url("non_detections", oid), result_format=format ) return q.result(index, sort) + + def query_forced_photometry(self, oid, format="json", index=None, sort=None): + """ + Gets all forced photometry detections of a given object + + Parameters + ---------- + oid : str + The object identifier + format : str + Return format. Can be one of 'pandas' | 'votable' | 'json' + """ + q = self._request( + "GET", "https://api.alerce.online/v2/lightcurve/forced-photometry/%s" % oid, result_format=format + ) + + # NOTA: la api principal de ztf no tiene ruta de forced photometry, la v2 si tiene. Esto es lo mas facil + # pero no es correcto. + + return q.result(index, sort) def query_magstats(self, oid, format="json", index=None, sort=None): """ diff --git a/tests/test_search.py b/tests/test_search.py index 5f193c0..4d36184 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -74,7 +74,11 @@ def mock_result(): mock_request.return_value.status_code = 200 mock_request.return_value.json = mock_result - sort = "mjd" + sort = "mjd"@patch.object(Session, "request") +def test_query_detections(mock_request): + mock_request.return_value.status_code = 200 + r = alerce.query_detections("oid") + assert r is not None r = alerce.query_objects(format="pandas", sort=sort) assert r.mjd.iloc[0] < r.mjd.iloc[1] @@ -157,7 +161,7 @@ def test_query_features(mock_request): @patch.object(Session, "request") -def test_query_single_feature(mock_request): +def test_query_single_feature(mock_request):ZTF24aalvzza mock_request.return_value.status_code = 200 r = alerce.query_feature(oid="oid", name="feature") assert r is not None @@ -175,3 +179,18 @@ def test_query_classes(mock_request): mock_request.return_value.status_code = 200 r = alerce.query_classes("lc_classifier", "bulk_0.0.1") assert r is not None + + +@patch.object(Session, "request") +def test_query_detections(mock_request): + def mock_result(): + return [ + {"candid":"candid1","tid":"ztf","sid":None,"aid":None,"pid":1234,"oid":"oid"}, + {"candid":"candid2","tid":"ztf","sid":None,"aid":None,"pid":1234,"oid":"oid"} + ] + + mock_request.return_value.status_code = 200 + mock_request.return_value.json = mock_result() + + r = alerce.query_detections("oid") + assert r is not None \ No newline at end of file From c680d39b808e6e1e36771f832661b4ed10f7755d Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 15:06:37 -0300 Subject: [PATCH 02/15] fixed fat finger error --- tests/test_search.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index 4d36184..81eb7b0 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -74,11 +74,7 @@ def mock_result(): mock_request.return_value.status_code = 200 mock_request.return_value.json = mock_result - sort = "mjd"@patch.object(Session, "request") -def test_query_detections(mock_request): - mock_request.return_value.status_code = 200 - r = alerce.query_detections("oid") - assert r is not None + sort = "mjd" r = alerce.query_objects(format="pandas", sort=sort) assert r.mjd.iloc[0] < r.mjd.iloc[1] @@ -161,7 +157,7 @@ def test_query_features(mock_request): @patch.object(Session, "request") -def test_query_single_feature(mock_request):ZTF24aalvzza +def test_query_single_feature(mock_request): mock_request.return_value.status_code = 200 r = alerce.query_feature(oid="oid", name="feature") assert r is not None @@ -180,7 +176,6 @@ def test_query_classes(mock_request): r = alerce.query_classes("lc_classifier", "bulk_0.0.1") assert r is not None - @patch.object(Session, "request") def test_query_detections(mock_request): def mock_result(): From 0874019d93a429229345065a82a0af407a6f86f0 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 15:17:47 -0300 Subject: [PATCH 03/15] trying to fix actions --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2785743..e58eb42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,7 @@ jobs: with: options: "--check --verbose" src: "alerce/" + version: "~=24.0" - name: Test with pytest run: | coverage run --source alerce/ -m pytest tests/ From 570688d6ce4dbd4c9bd1af05f652dde408c048b4 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 15:19:14 -0300 Subject: [PATCH 04/15] chore: runned black --- alerce/search.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/alerce/search.py b/alerce/search.py index 571c764..d00a3af 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -159,7 +159,7 @@ def query_non_detections(self, oid, format="json", index=None, sort=None): "GET", self.__get_url("non_detections", oid), result_format=format ) return q.result(index, sort) - + def query_forced_photometry(self, oid, format="json", index=None, sort=None): """ Gets all forced photometry detections of a given object @@ -172,12 +172,14 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): Return format. Can be one of 'pandas' | 'votable' | 'json' """ q = self._request( - "GET", "https://api.alerce.online/v2/lightcurve/forced-photometry/%s" % oid, result_format=format + "GET", + "https://api.alerce.online/v2/lightcurve/forced-photometry/%s" % oid, + result_format=format, ) # NOTA: la api principal de ztf no tiene ruta de forced photometry, la v2 si tiene. Esto es lo mas facil # pero no es correcto. - + return q.result(index, sort) def query_magstats(self, oid, format="json", index=None, sort=None): From 91aab445e4e46e227c47bb34a7a49de8a09a6470 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 15:22:14 -0300 Subject: [PATCH 05/15] using correct function --- tests/test_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_search.py b/tests/test_search.py index 81eb7b0..4bcd91d 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -187,5 +187,5 @@ def mock_result(): mock_request.return_value.status_code = 200 mock_request.return_value.json = mock_result() - r = alerce.query_detections("oid") + r = alerce.query_forced_photometry("oid") assert r is not None \ No newline at end of file From 0ed8478f37a6b0eeb2581e0ba4d78af8a7aec735 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 15:23:50 -0300 Subject: [PATCH 06/15] mocking json response correctly --- tests/test_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index 4bcd91d..e29a307 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -182,10 +182,10 @@ def mock_result(): return [ {"candid":"candid1","tid":"ztf","sid":None,"aid":None,"pid":1234,"oid":"oid"}, {"candid":"candid2","tid":"ztf","sid":None,"aid":None,"pid":1234,"oid":"oid"} - ] + ] mock_request.return_value.status_code = 200 - mock_request.return_value.json = mock_result() + mock_request.return_value.json = mock_result r = alerce.query_forced_photometry("oid") assert r is not None \ No newline at end of file From 10d703c9d254a0ce52e618a4b9bfdf39b12a7eb8 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Mon, 16 Dec 2024 16:43:20 -0300 Subject: [PATCH 07/15] modified command --- alerce/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerce/search.py b/alerce/search.py index d00a3af..cf29bf3 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -162,7 +162,7 @@ def query_non_detections(self, oid, format="json", index=None, sort=None): def query_forced_photometry(self, oid, format="json", index=None, sort=None): """ - Gets all forced photometry detections of a given object + Gets all forced photometry epochs of a given object Parameters ---------- From e56a697b8fcb4d375ce1adf7fe207a0c3c29920e Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Tue, 17 Dec 2024 09:30:24 -0300 Subject: [PATCH 08/15] documenting forced photometry model, missing descriptions --- docs/source/models/fored_photometry.rst | 130 ++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 docs/source/models/fored_photometry.rst diff --git a/docs/source/models/fored_photometry.rst b/docs/source/models/fored_photometry.rst new file mode 100644 index 0000000..e9a38b1 --- /dev/null +++ b/docs/source/models/fored_photometry.rst @@ -0,0 +1,130 @@ +Forced Photometry Response +========================= + +.. list-table:: Forced Photometry Response Model + :widths: 20 10 70 + :header-rows: 1 + + * - Name + - Type + - Description + * - pid + - integer + - Description + * - mjd + - number + - Description + * - fid + - integer + - Description + * - ra + - number + - Description + * - dec + - number + - Description + * - e_ra + - number + - Description + * - e_dec + - number + - Description + * - mag + - number + - Description + * - e_mag + - number + - Description + * - e_mag_corr + - number + - Description + * - e_mag_corr_ext + - number + - Description + * - isdiffpos + - integer + - Description + * - corrected + - boolean + - Description + * - dubious + - boolean + - Description + * - parent_candid + - number + - Description + * - has_stamp + - boolean + - Description + * - field + - integer + - Description + * - rcid + - integer + - Description + * - rfid + - integer + - Description + * - sciinpseeing + - number + - Description + * - scibckgnd + - number + - Description + * - scisigpix + - number + - Description + * - magzpsci + - number + - Description + * - magzpsciunc + - number + - Description + * - magzpscirms + - number + - Description + * - clrcoeff + - number + - Description + * - clrcounc + - number + - Description + * - exptime + - number + - Description + * - adpctdif1 + - number + - Description + * - adpctdif2 + - number + - Description + * - diffmaglim + -number + - Description + * - programid + - integer + - Description + * - procstatus + - string + - Description + * - distnr + - number + - Description + * - ranr + - number + - Description + * - decnr + - number + - Description + * - magnr + - number + - Description + * - sigmagnr + - number + - Description + * - chinr + - number + - Description + * - sharpnr + - number + - Description \ No newline at end of file From eb04728dce43db6a626325e30b59d60e44063b9f Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Tue, 17 Dec 2024 10:07:00 -0300 Subject: [PATCH 09/15] added some desriptions --- docs/source/models/fored_photometry.rst | 52 ++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/source/models/fored_photometry.rst b/docs/source/models/fored_photometry.rst index e9a38b1..45f56c9 100644 --- a/docs/source/models/fored_photometry.rst +++ b/docs/source/models/fored_photometry.rst @@ -10,13 +10,13 @@ Forced Photometry Response - Description * - pid - integer - - Description + - Processing ID for image * - mjd - number - Description * - fid - integer - - Description + - Filter ID (1=g; 2=R; 3=i) * - ra - number - Description @@ -61,70 +61,70 @@ Forced Photometry Response - Description * - rcid - integer - - Description + - Readout channel ID [00 .. 63] * - rfid - integer - - Description + - Processing ID for reference image to facilitate archive retrieval * - sciinpseeing - number - - Description + - Effective FWHM of sci image [pixels] * - scibckgnd - number - - Description + - Background level in sci image [DN] * - scisigpix - number - - Description + - Robust sigma per pixel in sci image [DN] * - magzpsci - number - - Description + - Magnitude zero point for photometry estimates [mag] * - magzpsciunc - number - - Description + - DescriptionMagnitude zero point uncertainty (in magzpsci) [mag] * - magzpscirms - number - - Description + - RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag] * - clrcoeff - number - - Description + - Color coefficient from linear fit from photometric calibration of science image * - clrcounc - number - - Description + - Color coefficient uncertainty from linear fit (corresponding to clrcoeff) * - exptime - number - - Description + - Integration time of camera exposure [sec] * - adpctdif1 - number - - Description + - Full sci image astrometric RMS along R.A. with respect to Gaia1 [arcsec] * - adpctdif2 - number - - Description + - Full sci image astrometric RMS along Dec. with respect to Gaia1 [arcsec] * - diffmaglim - -number - - Description + - number + - Expected 5-sigma mag limit in difference image based on global noise estimate [mag] * - programid - integer - - Description + - Program ID: encodes either public, collab, or caltech mode * - procstatus - string - - Description + - Forced photometry processing status codes (0 => no warnings); see documentation * - distnr - number - - Description + - distance to nearest source in reference image PSF-catalog [arcsec] * - ranr - number - - Description + - Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg] * - decnr - number - - Description + - Declination of nearest source in reference image PSF-catalog; J2000 [deg] * - magnr - number - - Description + - magnitude of nearest source in reference image PSF-catalog [mag] * - sigmagnr - number - - Description + - 1-sigma uncertainty in magnr [mag] * - chinr - number - - Description + - DAOPhot chi parameter of nearest source in reference image PSF-catalog * - sharpnr - number - - Description \ No newline at end of file + - DAOPhot sharp parameter of nearest source in reference image PSF-catalog \ No newline at end of file From 4f8456c954e39acf7eaa4c20330103ce1619618d Mon Sep 17 00:00:00 2001 From: Ale M Date: Tue, 17 Dec 2024 16:26:41 -0300 Subject: [PATCH 10/15] Added missing descriptions --- docs/source/models/detection.rst | 2 +- ...d_photometry.rst => forced_photometry.rst} | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) rename docs/source/models/{fored_photometry.rst => forced_photometry.rst} (71%) diff --git a/docs/source/models/detection.rst b/docs/source/models/detection.rst index 8c36f67..6c724dd 100644 --- a/docs/source/models/detection.rst +++ b/docs/source/models/detection.rst @@ -40,7 +40,7 @@ Detection Response - Magnitude from PSF-fit photometry [mag] * - sigmapsf - number - - 1-sigma uncertainty in magap [mag] + - 1-sigma uncertainty in magpsf [mag] * - magap - number - Aperture mag using 8 pixel diameter aperture [mag] diff --git a/docs/source/models/fored_photometry.rst b/docs/source/models/forced_photometry.rst similarity index 71% rename from docs/source/models/fored_photometry.rst rename to docs/source/models/forced_photometry.rst index 45f56c9..205f779 100644 --- a/docs/source/models/fored_photometry.rst +++ b/docs/source/models/forced_photometry.rst @@ -13,52 +13,55 @@ Forced Photometry Response - Processing ID for image * - mjd - number - - Description + - Modified julian Date * - fid - integer - - Filter ID (1=g; 2=R; 3=i) + - Filter ID (1=g; 2=r; 3=i) * - ra - number - - Description + - Right Ascension of epoch (same as of candidate) [deg] * - dec - number - - Description + - Declination of epoch (same as of candidate) [deg] * - e_ra - number - - Description + - Right Ascension uncertainty * - e_dec - number - - Description + - Declination uncertainty * - mag - number - - Description + - Magnitude from PSF-fit photometry [mag], obtained from magzpsci and forcediffimflux avro fields * - e_mag - number - - Description + - 1-sigma uncertainty in mag [mag], obtained from forcediffimflux and forcediffimfluxunc avro fields + * - mag_corr + - number + - Corrected PSF magnitude * - e_mag_corr - number - - Description + - Error of the corrected PSF magnitude assuming no contribution from an extended component * - e_mag_corr_ext - number - - Description + - Error of the corrected PSF magnitude assuming a contribution from an extended component * - isdiffpos - integer - - Description + - 1: candidate is from positive (sci minus ref) subtraction; -1: didate is from negative (ref minus sci) subtraction * - corrected - boolean - - Description + - Whether the corrected photometry was applied * - dubious - boolean - - Description + - Whether the corrected photometry should be flagged * - parent_candid - number - - Description + - candid of the alert for which forced photometry was triggered * - has_stamp - boolean - - Description + - Whether the epoch has an associated image stamp triplet (always False for forced photometry) * - field - integer - - Description + - ZTF field ID * - rcid - integer - Readout channel ID [00 .. 63] @@ -79,7 +82,7 @@ Forced Photometry Response - Magnitude zero point for photometry estimates [mag] * - magzpsciunc - number - - DescriptionMagnitude zero point uncertainty (in magzpsci) [mag] + - Magnitude zero point uncertainty (in magzpsci) [mag] * - magzpscirms - number - RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag] From 13a767e9b54eef96877071d8fb325ea57d3fcf2b Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Wed, 18 Dec 2024 11:35:03 -0300 Subject: [PATCH 11/15] added some extra code to expand extra fields --- alerce/search.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/alerce/search.py b/alerce/search.py index cf29bf3..5ce4e03 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -180,7 +180,13 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): # NOTA: la api principal de ztf no tiene ruta de forced photometry, la v2 si tiene. Esto es lo mas facil # pero no es correcto. - return q.result(index, sort) + # all this extra code is to expand the extra fields. + complete_result = q.result(index, sort) + extra_fields = complete_result.pop("extra_fields", None) + parsed_result = complete_result.update(extra_fields) + + return parsed_result + def query_magstats(self, oid, format="json", index=None, sort=None): """ From 0e2e2055f7b74b8bb0f8acec755d7df18a12ad75 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 19 Dec 2024 14:57:36 -0300 Subject: [PATCH 12/15] workarround to expand extrafields --- alerce/search.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/alerce/search.py b/alerce/search.py index 5ce4e03..641f13a 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -181,9 +181,26 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): # pero no es correcto. # all this extra code is to expand the extra fields. - complete_result = q.result(index, sort) - extra_fields = complete_result.pop("extra_fields", None) - parsed_result = complete_result.update(extra_fields) + if format == "json": + complete_result = q.result(index, sort) + parsed_result = [] + print(complete_result) + for result in complete_result: + new_result = result.copy() + extra_fields = new_result.pop("extra_fields", {}) + new_result.update(extra_fields) + parsed_result.append(new_result) + if format == "pandas" or format == "csv": + complete_result = q.result(index, sort) + extra_fields = complete_result["extra_fields"].copy() + complete_result = complete_result.drop(columns="extra_fields") + # expand + import pandas as pd + extra_fields = pd.json_normalize(extra_fields) + # merge + parsed_result = complete_result.merge(extra_fields) + if format == "csv": + parsed_result = parsed_result.to_csv(index=False) return parsed_result From db985f2422956f38cf77441d3e02683c855b0dd4 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 19 Dec 2024 15:02:44 -0300 Subject: [PATCH 13/15] runned black --- alerce/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerce/search.py b/alerce/search.py index 641f13a..a04d7f1 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -196,6 +196,7 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): complete_result = complete_result.drop(columns="extra_fields") # expand import pandas as pd + extra_fields = pd.json_normalize(extra_fields) # merge parsed_result = complete_result.merge(extra_fields) @@ -204,7 +205,6 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): return parsed_result - def query_magstats(self, oid, format="json", index=None, sort=None): """ Gets magnitude statistics of a given object From 6ff34c552225b432867150496745445b363b3a7a Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 19 Dec 2024 15:13:53 -0300 Subject: [PATCH 14/15] remove fields from forced photometry --- alerce/search.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/alerce/search.py b/alerce/search.py index a04d7f1..c592836 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -181,19 +181,23 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): # pero no es correcto. # all this extra code is to expand the extra fields. + complete_result = q.result(index, sort) + + FIELDS_TO_REMOVE = ["extra_fields", "aid", "sid"] + if format == "json": - complete_result = q.result(index, sort) parsed_result = [] print(complete_result) for result in complete_result: new_result = result.copy() extra_fields = new_result.pop("extra_fields", {}) + for f_t_r in FIELDS_TO_REMOVE: + new_result.pop(f_t_r, None) new_result.update(extra_fields) parsed_result.append(new_result) if format == "pandas" or format == "csv": - complete_result = q.result(index, sort) extra_fields = complete_result["extra_fields"].copy() - complete_result = complete_result.drop(columns="extra_fields") + complete_result = complete_result.drop(columns=FIELDS_TO_REMOVE) # expand import pandas as pd From 3034823c0b3c57f70c3517c4ca0fd14851b0be27 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Fri, 20 Dec 2024 11:07:39 -0300 Subject: [PATCH 15/15] removed print --- alerce/search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alerce/search.py b/alerce/search.py index c592836..bd188f3 100644 --- a/alerce/search.py +++ b/alerce/search.py @@ -187,7 +187,6 @@ def query_forced_photometry(self, oid, format="json", index=None, sort=None): if format == "json": parsed_result = [] - print(complete_result) for result in complete_result: new_result = result.copy() extra_fields = new_result.pop("extra_fields", {})