Merge pull request #1910 from ASFHyP3/develop

Release v4.3.0
ASFHyP3 · Nov 2, 2023 · b59592b · b59592b
2 parents 899f128 + a86469d
commit b59592b
Show file tree

Hide file tree

Showing 18 changed files with 63 additions and 236 deletions.
diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml
@@ -28,15 +28,14 @@ jobs:
               job_spec/INSAR_ISCE_TEST.yml
               job_spec/INSAR_ISCE_BURST.yml
               job_spec/WATER_MAP.yml
-              job_spec/WATER_MAP_TEST.yml
               job_spec/WATER_MAP_EQ.yml
               job_spec/S1_CORRECTION_ITS_LIVE.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
     environment:

diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml
@@ -29,7 +29,7 @@ jobs:
             expanded_max_vcpus: 10000
             required_surplus: 0
             security_environment: JPL-public
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-a19-jpl
@@ -44,7 +44,7 @@ jobs:
             expanded_max_vcpus: 10000
             required_surplus: 0
             security_environment: JPL-public
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-tibet-jpl
@@ -59,7 +59,7 @@ jobs:
             expanded_max_vcpus: 0
             required_surplus: 0
             security_environment: JPL-public
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-nisar-jpl
@@ -74,7 +74,7 @@ jobs:
             expanded_max_vcpus: 1600
             required_surplus: 0
             security_environment: JPL-public
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-avo
@@ -89,7 +89,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-watermap
@@ -104,7 +104,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-streamflow
@@ -119,7 +119,7 @@ jobs:
             expanded_max_vcpus: 1600
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: azdwr-hyp3
@@ -134,7 +134,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-cargill
@@ -149,7 +149,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-bgc-engineering
@@ -164,7 +164,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-carter
@@ -179,7 +179,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
           - environment: hyp3-pdc
@@ -196,7 +196,7 @@ jobs:
             expanded_max_vcpus: 640
             required_surplus: 0
             security_environment: ASF
-            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
             distribution_url: ''
 
     environment:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.3.0]
+### Changed
+- The `iterative_min_size` and `minimization_metric` parameters have been moved from the `WATER_MAP_TEST` job spec to the `WATER_MAP` job spec. The default `minimization_metric` value has been changed from `fmi` to `ts`.
+- The `known_water_threshold` parameter for the `WATER_MAP` job type is now nullable, with a default value of `null` instead of `30.0` percent. A water threshold is computed when the value is `null`.
+- Use Amazon Linux 2023 AMI in non-Earthdata Cloud environments
+  - Reduced the memory reservation of some job types due to slightly less memory being available for AWS Batch jobs on the AL2023 AMI
+- All deployments now use the `SPOT_PRICE_CAPACITY_OPTIMIZED` allocation strategy for AWS Batch. This includes JPL
+  deployments, reverting the temporary change to On Demand instances in HyP3 v3.10.8
+### Removed
+- The `WATER_MAP_TEST` job spec
+
 ## [4.2.1]
 ### Changed
 - The `ami_id` for EDC platforms now uses the original AMI.

diff --git a/apps/compute-cf.yml.j2 b/apps/compute-cf.yml.j2
@@ -57,8 +57,9 @@ Resources:
             --==BOUNDARY==
             Content-Type: text/cloud-boothook; charset="us-ascii"
 
+            #!/bin/bash
             cloud-init-per instance mkfs_ssd mkfs.ext4 /dev/nvme1n1
-            cloud-init-per instance mount_ssd mount /dev/nvme1n1 /var/lib/docker
+            mount /dev/nvme1n1 /var/lib/docker
 
             --==BOUNDARY==--
 
@@ -68,13 +69,8 @@ Resources:
       ServiceRole: !GetAtt BatchServiceRole.Arn
       Type: MANAGED
       ComputeResources:
-        {% if security_environment in ('JPL', 'JPL-public') %}
-        Type: EC2
-        AllocationStrategy: BEST_FIT_PROGRESSIVE
-        {% else %}
         Type: SPOT
-        AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
-        {% endif %}
+        AllocationStrategy: SPOT_PRICE_CAPACITY_OPTIMIZED
         MinvCpus: 0
         MaxvCpus: !Ref MaxvCpus
         InstanceTypes: !Ref InstanceTypes

diff --git a/apps/main-cf.yml.j2 b/apps/main-cf.yml.j2
@@ -46,7 +46,7 @@ Parameters:
 
   AmiId:
     Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
-    Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+    Default: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
 
   DefaultMaxvCpus:
     Description: Default maximum size for the AWS Batch compute environment

diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2
@@ -103,10 +103,6 @@
                 {
                   "Variable": "$.job_type",
                   "StringEquals": "WATER_MAP_EQ"
-                },
-                {
-                  "Variable": "$.job_type",
-                  "StringEquals": "WATER_MAP_TEST"
                 }
               ]
             },
@@ -138,7 +134,7 @@
         "ResourceRequirements": [
           {
             "Type": "MEMORY",
-            "Value": "7900"
+            "Value": "7875"
           }
         ]
       },

diff --git a/job_spec/AUTORIFT.yml b/job_spec/AUTORIFT.yml
@@ -51,7 +51,7 @@ AUTORIFT:
         - Ref::granules
       timeout: 10800
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
diff --git a/job_spec/AUTORIFT_ITS_LIVE.yml b/job_spec/AUTORIFT_ITS_LIVE.yml
@@ -54,7 +54,7 @@ AUTORIFT:
         - Ref::granules
       timeout: 10800
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
diff --git a/job_spec/AUTORIFT_ITS_LIVE_TEST.yml b/job_spec/AUTORIFT_ITS_LIVE_TEST.yml
@@ -55,7 +55,7 @@ AUTORIFT_TEST:
         - Ref::granules
       timeout: 10800
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
diff --git a/job_spec/INSAR_GAMMA.yml b/job_spec/INSAR_GAMMA.yml
@@ -105,7 +105,7 @@ INSAR_GAMMA:
         - Ref::granules
       timeout: 10800
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
diff --git a/job_spec/RIVER_WIDTH.yml b/job_spec/RIVER_WIDTH.yml
@@ -89,4 +89,4 @@ RIVER_WIDTH:
         - Ref::bucket_prefix
       timeout: 36000
       vcpu: 1
-      memory: 31600
+      memory: 31500
diff --git a/job_spec/RTC_GAMMA.yml b/job_spec/RTC_GAMMA.yml
@@ -127,7 +127,7 @@ RTC_GAMMA:
         - Ref::granules
       timeout: 36000
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
diff --git a/job_spec/WATER_MAP.yml b/job_spec/WATER_MAP.yml
@@ -83,9 +83,10 @@ WATER_MAP:
         type: number
     known_water_threshold:
       api_schema:
-        description: Threshold for extracting known water area in percent. Ignored when flood_depth_estimator is None.
-        default: 30.0
+        description: Threshold for extracting known water area in percent. A water threshold is computed when value is null. Ignored when flood_depth_estimator is None.
+        default: null
         type: number
+        nullable: true
     iterative_min:
       api_schema:
         description: Minimum bound used for iterative method. Ignored when flood_depth_estimator is None.
@@ -96,6 +97,20 @@ WATER_MAP:
         description: Maximum bound used for iterative method. Ignored when flood_depth_estimator is None.
         default: 15
         type: integer
+    iterative_min_size:
+      api_schema:
+        description: Minimum size of a connected waterbody in pixels for calculating flood depths with the iterative estimator
+        default: 0
+        type: integer
+        minimum: 0
+    minimization_metric:
+      api_schema:
+        description: Evaluation method to minimize during the iterative flood depth calculation. Options include a Fowlkes-Mallows index (fmi) or a threat score (ts). Only used when flood_depth_estimator is iterative.
+        default: ts
+        type: string
+        enum:
+          - fmi
+          - ts
   validators:
     - check_dem_coverage
   tasks:
@@ -133,7 +148,7 @@ WATER_MAP:
         - Ref::granules
       timeout: 36000
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
@@ -178,6 +193,10 @@ WATER_MAP:
         - Ref::iterative_min
         - --iterative-max
         - Ref::iterative_max
+        - --iterative-min-size
+        - Ref::iterative_min_size
+        - --minimization-metric
+        - Ref::minimization_metric
       timeout: 86400
       vcpu: 1
       memory: 126000
diff --git a/job_spec/WATER_MAP_EQ.yml b/job_spec/WATER_MAP_EQ.yml
@@ -90,7 +90,7 @@ WATER_MAP_EQ:
         - Ref::granules
       timeout: 36000
       vcpu: 1
-      memory: 31600
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD