diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml index 55f4974ee..b8a332552 100644 --- a/.github/workflows/deploy-enterprise-test.yml +++ b/.github/workflows/deploy-enterprise-test.yml @@ -28,7 +28,6 @@ jobs: job_spec/INSAR_ISCE_TEST.yml job_spec/INSAR_ISCE_BURST.yml job_spec/WATER_MAP.yml - job_spec/WATER_MAP_TEST.yml job_spec/WATER_MAP_EQ.yml job_spec/S1_CORRECTION_ITS_LIVE.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge @@ -36,7 +35,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' environment: diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index a4d55772b..1c15fb68d 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -29,7 +29,7 @@ jobs: expanded_max_vcpus: 10000 required_surplus: 0 security_environment: JPL-public - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-a19-jpl @@ -44,7 +44,7 @@ jobs: expanded_max_vcpus: 10000 required_surplus: 0 security_environment: JPL-public - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-tibet-jpl @@ -59,7 +59,7 @@ jobs: expanded_max_vcpus: 0 required_surplus: 0 security_environment: JPL-public - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-nisar-jpl @@ -74,7 +74,7 @@ jobs: expanded_max_vcpus: 1600 required_surplus: 0 security_environment: JPL-public - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-avo @@ -89,7 +89,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-watermap @@ -104,7 +104,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-streamflow @@ -119,7 +119,7 @@ jobs: expanded_max_vcpus: 1600 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: azdwr-hyp3 @@ -134,7 +134,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-cargill @@ -149,7 +149,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-bgc-engineering @@ -164,7 +164,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-carter @@ -179,7 +179,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' - environment: hyp3-pdc @@ -196,7 +196,7 @@ jobs: expanded_max_vcpus: 640 required_surplus: 0 security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id distribution_url: '' environment: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5be11bea5..eac8d457b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.3.0] +### Changed +- The `iterative_min_size` and `minimization_metric` parameters have been moved from the `WATER_MAP_TEST` job spec to the `WATER_MAP` job spec. The default `minimization_metric` value has been changed from `fmi` to `ts`. +- The `known_water_threshold` parameter for the `WATER_MAP` job type is now nullable, with a default value of `null` instead of `30.0` percent. A water threshold is computed when the value is `null`. +- Use Amazon Linux 2023 AMI in non-Earthdata Cloud environments + - Reduced the memory reservation of some job types due to slightly less memory being available for AWS Batch jobs on the AL2023 AMI +- All deployments now use the `SPOT_PRICE_CAPACITY_OPTIMIZED` allocation strategy for AWS Batch. This includes JPL + deployments, reverting the temporary change to On Demand instances in HyP3 v3.10.8 +### Removed +- The `WATER_MAP_TEST` job spec + ## [4.2.1] ### Changed - The `ami_id` for EDC platforms now uses the original AMI. diff --git a/apps/compute-cf.yml.j2 b/apps/compute-cf.yml.j2 index 48721a7f4..0cb23e152 100644 --- a/apps/compute-cf.yml.j2 +++ b/apps/compute-cf.yml.j2 @@ -57,8 +57,9 @@ Resources: --==BOUNDARY== Content-Type: text/cloud-boothook; charset="us-ascii" + #!/bin/bash cloud-init-per instance mkfs_ssd mkfs.ext4 /dev/nvme1n1 - cloud-init-per instance mount_ssd mount /dev/nvme1n1 /var/lib/docker + mount /dev/nvme1n1 /var/lib/docker --==BOUNDARY==-- @@ -68,13 +69,8 @@ Resources: ServiceRole: !GetAtt BatchServiceRole.Arn Type: MANAGED ComputeResources: - {% if security_environment in ('JPL', 'JPL-public') %} - Type: EC2 - AllocationStrategy: BEST_FIT_PROGRESSIVE - {% else %} Type: SPOT - AllocationStrategy: SPOT_CAPACITY_OPTIMIZED - {% endif %} + AllocationStrategy: SPOT_PRICE_CAPACITY_OPTIMIZED MinvCpus: 0 MaxvCpus: !Ref MaxvCpus InstanceTypes: !Ref InstanceTypes diff --git a/apps/main-cf.yml.j2 b/apps/main-cf.yml.j2 index e23f47cf9..260c5390e 100644 --- a/apps/main-cf.yml.j2 +++ b/apps/main-cf.yml.j2 @@ -46,7 +46,7 @@ Parameters: AmiId: Type: AWS::SSM::Parameter::Value - Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + Default: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id DefaultMaxvCpus: Description: Default maximum size for the AWS Batch compute environment diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 9d04675f7..17bb8f63e 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -103,10 +103,6 @@ { "Variable": "$.job_type", "StringEquals": "WATER_MAP_EQ" - }, - { - "Variable": "$.job_type", - "StringEquals": "WATER_MAP_TEST" } ] }, @@ -138,7 +134,7 @@ "ResourceRequirements": [ { "Type": "MEMORY", - "Value": "7900" + "Value": "7875" } ] }, diff --git a/job_spec/AUTORIFT.yml b/job_spec/AUTORIFT.yml index 1619248ff..5f898a55a 100644 --- a/job_spec/AUTORIFT.yml +++ b/job_spec/AUTORIFT.yml @@ -51,7 +51,7 @@ AUTORIFT: - Ref::granules timeout: 10800 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/AUTORIFT_ITS_LIVE.yml b/job_spec/AUTORIFT_ITS_LIVE.yml index 2ecd4c852..e60b05f14 100644 --- a/job_spec/AUTORIFT_ITS_LIVE.yml +++ b/job_spec/AUTORIFT_ITS_LIVE.yml @@ -54,7 +54,7 @@ AUTORIFT: - Ref::granules timeout: 10800 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/AUTORIFT_ITS_LIVE_TEST.yml b/job_spec/AUTORIFT_ITS_LIVE_TEST.yml index 3a7616d02..04a2088ea 100644 --- a/job_spec/AUTORIFT_ITS_LIVE_TEST.yml +++ b/job_spec/AUTORIFT_ITS_LIVE_TEST.yml @@ -55,7 +55,7 @@ AUTORIFT_TEST: - Ref::granules timeout: 10800 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/INSAR_GAMMA.yml b/job_spec/INSAR_GAMMA.yml index 21d1823b4..afe5edc66 100644 --- a/job_spec/INSAR_GAMMA.yml +++ b/job_spec/INSAR_GAMMA.yml @@ -105,7 +105,7 @@ INSAR_GAMMA: - Ref::granules timeout: 10800 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/RIVER_WIDTH.yml b/job_spec/RIVER_WIDTH.yml index 3fddb9465..ed44fc3bf 100644 --- a/job_spec/RIVER_WIDTH.yml +++ b/job_spec/RIVER_WIDTH.yml @@ -89,4 +89,4 @@ RIVER_WIDTH: - Ref::bucket_prefix timeout: 36000 vcpu: 1 - memory: 31600 + memory: 31500 diff --git a/job_spec/RTC_GAMMA.yml b/job_spec/RTC_GAMMA.yml index b6a47a13f..130045d7d 100644 --- a/job_spec/RTC_GAMMA.yml +++ b/job_spec/RTC_GAMMA.yml @@ -127,7 +127,7 @@ RTC_GAMMA: - Ref::granules timeout: 36000 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/WATER_MAP.yml b/job_spec/WATER_MAP.yml index 7ea06e9f6..74103b04b 100644 --- a/job_spec/WATER_MAP.yml +++ b/job_spec/WATER_MAP.yml @@ -83,9 +83,10 @@ WATER_MAP: type: number known_water_threshold: api_schema: - description: Threshold for extracting known water area in percent. Ignored when flood_depth_estimator is None. - default: 30.0 + description: Threshold for extracting known water area in percent. A water threshold is computed when value is null. Ignored when flood_depth_estimator is None. + default: null type: number + nullable: true iterative_min: api_schema: description: Minimum bound used for iterative method. Ignored when flood_depth_estimator is None. @@ -96,6 +97,20 @@ WATER_MAP: description: Maximum bound used for iterative method. Ignored when flood_depth_estimator is None. default: 15 type: integer + iterative_min_size: + api_schema: + description: Minimum size of a connected waterbody in pixels for calculating flood depths with the iterative estimator + default: 0 + type: integer + minimum: 0 + minimization_metric: + api_schema: + description: Evaluation method to minimize during the iterative flood depth calculation. Options include a Fowlkes-Mallows index (fmi) or a threat score (ts). Only used when flood_depth_estimator is iterative. + default: ts + type: string + enum: + - fmi + - ts validators: - check_dem_coverage tasks: @@ -133,7 +148,7 @@ WATER_MAP: - Ref::granules timeout: 36000 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD @@ -178,6 +193,10 @@ WATER_MAP: - Ref::iterative_min - --iterative-max - Ref::iterative_max + - --iterative-min-size + - Ref::iterative_min_size + - --minimization-metric + - Ref::minimization_metric timeout: 86400 vcpu: 1 memory: 126000 diff --git a/job_spec/WATER_MAP_EQ.yml b/job_spec/WATER_MAP_EQ.yml index 1840df614..d29496dc1 100644 --- a/job_spec/WATER_MAP_EQ.yml +++ b/job_spec/WATER_MAP_EQ.yml @@ -90,7 +90,7 @@ WATER_MAP_EQ: - Ref::granules timeout: 36000 vcpu: 1 - memory: 31600 + memory: 31500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/WATER_MAP_TEST.yml b/job_spec/WATER_MAP_TEST.yml deleted file mode 100644 index 7c707e0d0..000000000 --- a/job_spec/WATER_MAP_TEST.yml +++ /dev/null @@ -1,194 +0,0 @@ -WATER_MAP_TEST: - required_parameters: - - granules - parameters: - granules: - default: '""' - api_schema: - type: array - minItems: 1 - maxItems: 1 - items: - anyOf: - - description: The name of the IW VV+VH Sentinel-1 GRDH granule to process - type: string - pattern: "^S1[AB]_IW_GRDH_1SDV" - minLength: 67 - maxLength: 67 - example: S1A_IW_GRDH_1SDV_20210413T235641_20210413T235706_037439_0469D0_3F2B - - description: The name of the IW VV+VH Sentinel-1 SLC granule to process - type: string - pattern: "^S1[AB]_IW_SLC__1SDV" - minLength: 67 - maxLength: 67 - example: S1A_IW_SLC__1SDV_20211110T234815_20211110T234842_040516_04CE0A_E717 - bucket_prefix: - default: '""' - resolution: - api_schema: - default: 30.0 - description: Desired output pixel spacing in meters - type: number - enum: - - 30.0 - - 20.0 - - 10.0 - speckle_filter: - api_schema: - description: Apply an Enhanced Lee speckle filter - default: false - type: boolean - max_vv_threshold: - api_schema: - description: Maximum threshold value to use for VV polarized raster in decibels (dB) - default: -15.5 - type: number - max_vh_threshold: - api_schema: - description: Maximum threshold value to use for VH polarized raster in decibels (dB) - default: -23.0 - type: number - hand_threshold: - api_schema: - description: The maximum height above nearest drainage in meters to consider a pixel valid - default: 15.0 - type: number - hand_fraction: - api_schema: - description: The minimum fraction of valid HAND pixels required in a tile for thresholding - default: 0.8 - type: number - membership_threshold: - api_schema: - description: The average membership to the fuzzy indicators required for a water pixel - default: 0.45 - type: number - flood_depth_estimator: - api_schema: - description: Flood depth estimation approach. A value of null or None indicates that flood depth estimation will not be performed. - default: null - type: string - nullable: true - enum: - - iterative - - logstat - - nmad - - null - - numpy - - None - water_level_sigma: - api_schema: - description: Standard deviation to estimate max water height for each object. Ignored when flood_depth_estimator is None. - default: 3.0 - type: number - known_water_threshold: - api_schema: - description: Threshold for extracting known water area in percent. A water threshold is computed when value is null. Ignored when flood_depth_estimator is None. - default: null - type: number - nullable: true - iterative_min: - api_schema: - description: Minimum bound used for iterative method. Ignored when flood_depth_estimator is None. - default: 0 - type: integer - iterative_max: - api_schema: - description: Maximum bound used for iterative method. Ignored when flood_depth_estimator is None. - default: 15 - type: integer - minimization_metric: - api_schema: - description: Evaluation method to minimize during the iterative flood depth calculation. Options include a Fowlkes-Mallows index (fmi) or a threat score (ts). Only used when flood_depth_estimator is iterative. - default: fmi - type: string - enum: - - fmi - - ts - validators: - - check_dem_coverage - tasks: - - name: RTC - image: 845172464411.dkr.ecr.us-west-2.amazonaws.com/hyp3-gamma - command: - - ++process - - rtc - - ++omp-num-threads - - '4' - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::bucket_prefix - - --resolution - - Ref::resolution - - --speckle-filter - - Ref::speckle_filter - - --scale - - 'power' - - --radiometry - - 'gamma0' - - --dem-matching - - 'false' - - --include-dem - - 'true' - - --include-inc-map - - 'false' - - --include-scattering-area - - 'false' - - --include-rgb - - 'true' - - --dem-name - - 'copernicus' - - Ref::granules - timeout: 36000 - vcpu: 1 - memory: 31600 - secrets: - - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD - - name: '' - image: ghcr.io/asfhyp3/asf-tools - command: - - ++process - - water_map - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::bucket_prefix - - --max-vv-threshold - - Ref::max_vv_threshold - - --max-vh-threshold - - Ref::max_vh_threshold - - --hand-threshold - - Ref::hand_threshold - - --hand-fraction - - Ref::hand_fraction - - --membership-threshold - - Ref::membership_threshold - timeout: 36000 - vcpu: 1 - memory: 126000 - - name: FLOOD_MAP - image: ghcr.io/asfhyp3/asf-tools - command: - - ++process - - flood_map - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::bucket_prefix - - --estimator - - Ref::flood_depth_estimator - - --water-level-sigma - - Ref::water_level_sigma - - --known-water-threshold - - Ref::known_water_threshold - - --iterative-min - - Ref::iterative_min - - --iterative-max - - Ref::iterative_max - - --minimization-metric - - Ref::minimization_metric - timeout: 86400 - vcpu: 1 - memory: 126000 diff --git a/requirements-all.txt b/requirements-all.txt index 9a98d4921..cf9a35097 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -4,9 +4,9 @@ -r requirements-apps-start-execution-manager.txt -r requirements-apps-start-execution-worker.txt -r requirements-apps-update-db.txt -boto3==1.28.70 +boto3==1.28.75 jinja2==3.1.2 -moto[dynamodb]==4.2.6 +moto[dynamodb]==4.2.7 pytest==7.4.3 PyYAML==6.0.1 responses==0.23.3 @@ -16,4 +16,4 @@ flake8-blind-except==0.2.1 flake8-builtins==2.1.0 setuptools==68.2.2 openapi-spec-validator==0.4.0 -cfn-lint==0.83.0 +cfn-lint==0.83.1 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index 248d6db5a..cf3c233ee 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.28.70 +boto3==1.28.75 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index 03d863e93..11f7b8310 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.28.70 +boto3==1.28.75 ./lib/lambda_logging/