Merge pull request #1720 from ASFHyP3/develop

Release v3.9.5
ASFHyP3 · Jul 19, 2023 · f78ea57 · f78ea57
2 parents 563480c + 163436f
commit f78ea57
Show file tree

Hide file tree

Showing 14 changed files with 74 additions and 37 deletions.
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -13,6 +13,6 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.8.0
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml
@@ -0,0 +1,15 @@
+name: Create Jira issue
+
+on:
+  issues:
+    types: [labeled]
+
+jobs:
+  call-create-jira-issue-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    secrets:
+      JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+      JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+      JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }}
+      JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }}
diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml
@@ -102,6 +102,6 @@ jobs:
   call-bump-version-workflow:
     if: github.ref == 'refs/heads/main'
     needs: deploy
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.8.0
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml
@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.8.0
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.8.0
     with:
       release_prefix: HyP3
     secrets:

diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml
@@ -87,4 +87,4 @@ jobs:
           snyk iac test --severity-threshold=high
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.8.0
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.9.5]
+### Added
+- `apply_water_mask` option for `INSAR_ISCE_BURST` jobs
+### Changed
+- `POST /jobs` now returns HTTP 400 for Sentinel-1 Burst granules that do not exist in CMR
+- `POST /jobs` now returns HTTP 400 for INSAR_ISCE_BURST jobs for burst granules that do not intersect the
+  Copernicus GLO-30 Public DEM.
+
 ## [3.9.4]
 ### Fixed
 - Modified `start_execution_manager` to submit no more than 2 batches of 300 jobs, in order to reduce payload size.

diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py
@@ -42,7 +42,8 @@ def has_sufficient_coverage(granule: Polygon, buffer: float = 0.15, threshold: f
 
 def get_cmr_metadata(granules):
     cmr_parameters = {
-        'producer_granule_id': granules,
+        'granule_ur': [f'{granule}*' for granule in granules],
+        'options[granule_ur][pattern]': 'true',
         'provider': 'ASF',
         'short_name': [
             'SENTINEL-1A_SLC',
@@ -51,23 +52,23 @@ def get_cmr_metadata(granules):
             'SENTINEL-1B_SP_GRD_HIGH',
             'SENTINEL-1A_DP_GRD_HIGH',
             'SENTINEL-1B_DP_GRD_HIGH',
+            'SENTINEL-1_BURSTS',
         ],
         'page_size': 2000,
     }
     response = requests.post(CMR_URL, data=cmr_parameters)
     response.raise_for_status()
     granules = [
         {
-            'name': entry['producer_granule_id'],
+            'name': entry.get('producer_granule_id', entry.get('title')),
             'polygon': Polygon(format_points(entry['polygons'][0][0]))
         } for entry in response.json()['feed']['entry']
     ]
     return granules
 
 
 def is_third_party_granule(granule):
-    # FIXME remove the -BURST case when we integrate with prod CMR
-    return granule.startswith('S2') or granule.startswith('L') or granule.endswith('-BURST')
+    return granule.startswith('S2') or granule.startswith('L')
 
 
 def check_granules_exist(granules, granule_metadata):

diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml
@@ -20,6 +20,11 @@ INSAR_ISCE_BURST:
           example: S1_249434_IW1_20230523T170733_VV_8850-BURST
     bucket_prefix:
       default:  '""'
+    apply_water_mask:
+      api_schema:
+        description: Set pixels over coastal and large inland water bodies to NoData in wrapped and unwrapped phase GeoTIFFs (after unwrapping).
+        default: false
+        type: boolean
     looks:
       api_schema:
         description: Number of looks to take in range and azimuth
@@ -29,7 +34,8 @@ INSAR_ISCE_BURST:
           - 20x4
           - 10x2
           - 5x1
-  validators: []
+  validators:
+    - check_dem_coverage
   tasks:
     - name: ''
       image: ghcr.io/asfhyp3/hyp3-isce2
@@ -42,6 +48,8 @@ INSAR_ISCE_BURST:
         - '!Ref Bucket'
         - --bucket-prefix
         - Ref::bucket_prefix
+        - --apply-water-mask
+        - Ref::apply_water_mask
         - --looks
         - Ref::looks
         - Ref::granules

diff --git a/requirements-all.txt b/requirements-all.txt
@@ -6,7 +6,7 @@
 -r requirements-apps-start-execution-manager.txt
 -r requirements-apps-start-execution-worker.txt
 -r requirements-apps-update-db.txt
-boto3==1.26.154
+boto3==1.26.163
 jinja2==3.1.2
 moto[dynamodb]==4.1.11
 pytest==7.3.2
@@ -16,6 +16,6 @@ flake8==6.0.0
 flake8-import-order==0.18.2
 flake8-blind-except==0.2.1
 flake8-builtins==2.1.0
-setuptools==67.8.0
+setuptools==68.0.0
 openapi-spec-validator==0.4.0
-cfn-lint==0.77.9
+cfn-lint==0.77.10
diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt
@@ -1,5 +1,6 @@
 flask==2.2.5
 Flask-Cors==3.0.10
+jsonschema==4.17.3
 openapi-core==0.14.5
 prance==0.22.11.4.0
 PyJWT==2.7.0

diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt
@@ -1,3 +1,3 @@
-boto3==1.26.154
+boto3==1.26.163
 ./lib/dynamo/
 ./lib/lambda_logging/
diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt
@@ -1,2 +1,2 @@
-boto3==1.26.154
+boto3==1.26.163
 ./lib/lambda_logging/
diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py
@@ -1,8 +1,9 @@
+import responses
 from pytest import raises
 from shapely.geometry import Polygon
 from test_api.conftest import setup_requests_mock_with_given_polygons
 
-from hyp3_api import validation
+from hyp3_api import CMR_URL, validation
 
 
 def rectangle(north, south, east, west):
@@ -176,34 +177,37 @@ def test_is_third_party_granule():
     assert validation.is_third_party_granule('LC09_L1GT_215109_20220125_20220125_02_T2')
     assert validation.is_third_party_granule('LO09_L1GT_215109_20220210_20220210_02_T2')
     assert validation.is_third_party_granule('LT09_L1GT_215109_20220210_20220210_02_T2')
-    # FIXME remove the -BURST case when we integrate with prod CMR
-    assert validation.is_third_party_granule('S1_249434_IW1_20230523T170733_VV_8850-BURST')
+    assert not validation.is_third_party_granule('S1_249434_IW1_20230523T170733_VV_8850-BURST')
     assert not validation.is_third_party_granule('S1A_IW_SLC__1SSH_20150608T205059_20150608T205126_006287_0083E8_C4F0')
     assert not validation.is_third_party_granule('foo')
 
 
+@responses.activate
 def test_get_cmr_metadata():
-    granule = 'S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8'
-    fake_granule = 'not a real granule'
-    granules = [granule, fake_granule]
-
-    granule_polygon_pairs = [
-        (granule,
-         [['13.705972 -91.927132 14.452647 -91.773392 14.888498 -94.065727 '
-           '14.143632 -94.211563 13.705972 -91.927132']])
-    ]
-    setup_requests_mock_with_given_polygons(granule_polygon_pairs)
+    response_payload = {
+        'feed': {
+            'entry': [
+                {
+                    'producer_granule_id': 'foo',
+                    'polygons': [["-31.4 25.0 -29.7 25.5 -29.5 24.6 -31.2 24.1 -31.4 25.0"]],
+                },
+                {
+                    'title': 'bar',
+                    'polygons': [["0 1 2 3 4 5 6 7 0 1"]],
+                }
+            ],
+        },
+    }
+    responses.post(CMR_URL, json=response_payload)
 
-    assert validation.get_cmr_metadata(granules) == [
+    assert validation.get_cmr_metadata(['foo', 'bar', 'hello']) == [
+        {
+            'name': 'foo',
+            'polygon': Polygon([[25.0, -31.4], [25.5, -29.7], [24.6, -29.5], [24.1, -31.2]]),
+        },
         {
-            'name': granule,
-            'polygon': Polygon([
-                [-91.927132, 13.705972],
-                [-91.773392, 14.452647],
-                [-94.065727, 14.888498],
-                [-94.211563, 14.143632],
-                [-91.927132, 13.705972],
-            ]),
+            'name': 'bar',
+            'polygon': Polygon([[1, 0], [3, 2], [5, 4], [7, 6]]),
         },
     ]