Merge pull request #2140 from ASFHyP3/develop

Release v6.3.0
ASFHyP3 · Mar 12, 2024 · 017a121 · 017a121
2 parents 59bdd09 + 550b54f
commit 017a121
Show file tree

Hide file tree

Showing 15 changed files with 152 additions and 86 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [6.3.0]
+
+### Changed
+- `/costs` API endpoint now returns a list of job cost dictionaries, instead of a dictionary of dictionaries.
+- Cost table parameters are now contained within the `parameter_value` dictionary key.
+- Cost table costs are now contained within the `cost` dictionary key.
+
 ## [6.2.0]
 
 HyP3 is in the process of transitioning from a monthly job quota to a credits system. [HyP3 v6.0.0](https://github.com/ASFHyP3/hyp3/releases/tag/v6.0.0) implemented the new credits system without changing the number of jobs that users can run per month. This release implements the capability to assign a different credit cost to each type of job, again without actually changing the number of jobs that users can run per month.

diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2
@@ -25,7 +25,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/costs"
+                $ref: "#/components/schemas/costs_response"
 
   /jobs:
 
@@ -117,9 +117,34 @@ paths:
 components:
   schemas:
 
-    costs:
-      description: Table of job costs.
-      type: object
+    costs_response:
+      description: List of job costs.
+      type: array
+      items:
+        type: object
+        required:
+          - job_type
+        properties:
+          job_type:
+            $ref: "./job_parameters.yml#/components/schemas/job_type"
+          cost:
+            $ref: "#/components/schemas/credits"
+          cost_parameter:
+            type: string
+          cost_table:
+            type: array
+            items:
+              type: object
+              required:
+                - parameter_value
+                - cost
+              properties:
+                parameter_value:
+                  oneOf:
+                    - type: string
+                    - type: number
+                cost:
+                  $ref: "#/components/schemas/credits"
 
     post_jobs_body:
       description: List for new jobs to submit for processing.

diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py
@@ -11,7 +11,6 @@
 from hyp3_api.util import get_granules
 
 DEM_COVERAGE = None
-DEM_COVERAGE_LEGACY = None
 
 
 class GranuleValidationError(Exception):

diff --git a/apps/render_cf.py b/apps/render_cf.py
@@ -48,12 +48,15 @@ def render_default_params_by_job_type(job_types: dict) -> None:
 
 
 def render_costs(job_types: dict, cost_profile: str) -> None:
-    costs = {
-        job_type: job_spec['cost_profiles'][cost_profile]
+    costs = [
+        {
+            'job_type': job_type,
+            **job_spec['cost_profiles'][cost_profile],
+        }
         for job_type, job_spec in job_types.items()
-    }
-    with open(Path('lib') / 'dynamo' / 'dynamo' / 'costs.yml', 'w') as f:
-        yaml.safe_dump(costs, f)
+    ]
+    with open(Path('lib') / 'dynamo' / 'dynamo' / 'costs.json', 'w') as f:
+        json.dump(costs, f, indent=2)
 
 
 def main():

diff --git a/job_spec/INSAR_GAMMA.yml b/job_spec/INSAR_GAMMA.yml
@@ -74,8 +74,10 @@ INSAR_GAMMA:
     EDC:
       cost_parameter: looks
       cost_table:
-        20x4: 10.0
-        10x2: 15.0
+        - parameter_value: 20x4
+          cost: 10.0
+        - parameter_value: 10x2
+          cost: 15.0
     DEFAULT:
       cost: 1.0
   validators:

diff --git a/job_spec/RTC_GAMMA.yml b/job_spec/RTC_GAMMA.yml
@@ -93,9 +93,12 @@ RTC_GAMMA:
     EDC:
       cost_parameter: resolution
       cost_table:
-        30.0: 5.0
-        20.0: 15.0
-        10.0: 60.0
+        - parameter_value: 30.0
+          cost: 5.0
+        - parameter_value: 20.0
+          cost: 15.0
+        - parameter_value: 10.0
+          cost: 60.0
     DEFAULT:
       cost: 1.0
   validators:

diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py
@@ -6,14 +6,13 @@
 from typing import List, Optional
 from uuid import uuid4
 
-import yaml
 from boto3.dynamodb.conditions import Attr, Key
 
 import dynamo.user
 from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, format_time, get_request_time_expression
 
-costs_file = Path(__file__).parent / 'costs.yml'
-COSTS = convert_floats_to_decimals(yaml.safe_load(costs_file.read_text()))
+costs_file = Path(__file__).parent / 'costs.json'
+COSTS = convert_floats_to_decimals(json.loads(costs_file.read_text()))
 
 default_params_file = Path(__file__).parent / 'default_params_by_job_type.json'
 if default_params_file.exists():
@@ -99,18 +98,24 @@ def _prepare_job_for_database(
     return prepared_job
 
 
-def _get_credit_cost(job: dict, costs: dict) -> Decimal:
+def _get_credit_cost(job: dict, costs: list[dict]) -> Decimal:
     job_type = job['job_type']
-    cost_definition = costs[job_type]
-
-    if cost_definition.keys() not in ({'cost_parameter', 'cost_table'}, {'cost'}):
-        raise ValueError(f'Cost definition for job type {job_type} has invalid keys: {cost_definition.keys()}')
-
-    if 'cost_parameter' in cost_definition:
-        parameter_value = job['job_parameters'][cost_definition['cost_parameter']]
-        return cost_definition['cost_table'][parameter_value]
-
-    return cost_definition['cost']
+    for cost_definition in costs:
+        if cost_definition['job_type'] == job_type:
+
+            if cost_definition.keys() not in ({'job_type', 'cost_parameter', 'cost_table'}, {'job_type', 'cost'}):
+                raise ValueError(f'Cost definition for job type {job_type} has invalid keys: {cost_definition.keys()}')
+
+            if 'cost_parameter' in cost_definition:
+                cost_parameter = cost_definition['cost_parameter']
+                parameter_value = job['job_parameters'][cost_parameter]
+                for item in cost_definition['cost_table']:
+                    if item['parameter_value'] == parameter_value:
+                        return item['cost']
+                raise ValueError(f'Cost not found for job type {job_type} with {cost_parameter} == {parameter_value}')
+
+            return cost_definition['cost']
+    raise ValueError(f'Cost not found for job type {job_type}')
 
 
 def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_key=None):

diff --git a/lib/dynamo/setup.py b/lib/dynamo/setup.py
@@ -8,11 +8,10 @@
     install_requires=[
         'boto3',
         'python-dateutil',
-        'pyyaml',
     ],
     python_requires='~=3.9',
 
     packages=find_packages(),
 
-    package_data={'dynamo': ['*.json', '*.yml']},
+    package_data={'dynamo': ['*.json']},
 )
diff --git a/requirements-all.txt b/requirements-all.txt
@@ -5,10 +5,10 @@
 -r requirements-apps-start-execution-worker.txt
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-update-db.txt
-boto3==1.34.48
+boto3==1.34.60
 jinja2==3.1.3
-moto[dynamodb]==5.0.3.dev5
-pytest==8.0.1
+moto[dynamodb]==5.0.3.dev33
+pytest==8.1.1
 PyYAML==6.0.1
 responses==0.25.0
 flake8==7.0.0
@@ -17,4 +17,4 @@ flake8-blind-except==0.2.1
 flake8-builtins==2.2.0
 setuptools==69.1.1
 openapi-spec-validator==0.7.1
-cfn-lint==0.85.2
+cfn-lint==0.86.0
diff --git a/requirements-apps-api-binary.txt b/requirements-apps-api-binary.txt
@@ -1 +1 @@
-cryptography==42.0.4
+cryptography==42.0.5
diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt
@@ -1 +1 @@
-boto3==1.34.48
+boto3==1.34.60
diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt
@@ -1,3 +1,3 @@
-boto3==1.34.48
+boto3==1.34.60
 ./lib/dynamo/
 ./lib/lambda_logging/
diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt
@@ -1,2 +1,2 @@
-boto3==1.34.48
+boto3==1.34.60
 ./lib/lambda_logging/
diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py
@@ -51,14 +51,6 @@ def test_has_sufficient_coverage():
     poly = rectangle(40.1, 40, -126, -125.000140)
     assert not validation.has_sufficient_coverage(poly)
 
-    # minimum sufficient legacy coverage off the coast of Eureka, CA
-    poly = rectangle(40.1, 40, -126, -124.845)
-    assert validation.has_sufficient_coverage(poly)
-
-    # almost minimum sufficient legacy coverage off the coast of Eureka, CA
-    poly = rectangle(40.1, 40, -126, -124.849)
-    assert validation.has_sufficient_coverage(poly)
-
     # polygon in missing tile over Gulf of California
     poly = rectangle(26.9, 26.1, -110.1, -110.9)
     assert not validation.has_sufficient_coverage(poly)

diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py
@@ -183,19 +183,30 @@ def test_query_jobs_by_type(tables):
 
 
 def test_get_credit_cost():
-    costs = {
-        'RTC_GAMMA': {
+    costs = [
+        {
+            'job_type': 'RTC_GAMMA',
             'cost_parameter': 'resolution',
-            'cost_table': {
-                10: 60.0,
-                20: 15.0,
-                30: 5.0,
-            },
-        },
-        'INSAR_ISCE_BURST': {
+            'cost_table': [
+                {
+                    'parameter_value': 10.0,
+                    'cost': 60.0,
+                },
+                {
+                    'parameter_value': 20.0,
+                    'cost': 15.0,
+                },
+                {
+                    'parameter_value': 30.0,
+                    'cost': 5.0,
+                },
+            ],
+        },
+        {
+            'job_type': 'INSAR_ISCE_BURST',
             'cost': 1.0,
         }
-    }
+    ]
     assert dynamo.jobs._get_credit_cost(
         {'job_type': 'RTC_GAMMA', 'job_parameters': {'resolution': 10.0}},
         costs
@@ -208,7 +219,7 @@ def test_get_credit_cost():
         {'job_type': 'RTC_GAMMA', 'job_parameters': {'resolution': 30.0}},
         costs
     ) == 5.0
-    with pytest.raises(KeyError):
+    with pytest.raises(ValueError):
         dynamo.jobs._get_credit_cost(
             {'job_type': 'RTC_GAMMA', 'job_parameters': {'resolution': 13.0}},
             costs
@@ -224,14 +235,14 @@ def test_get_credit_cost():
 
 
 def test_get_credit_cost_validate_keys():
-    costs = {
-        'JOB_TYPE_A': {'cost_parameter': 'foo', 'cost_table': {'bar': 3.0}},
-        'JOB_TYPE_B': {'cost': 5.0},
-        'JOB_TYPE_C': {'cost_parameter': ''},
-        'JOB_TYPE_D': {'cost_table': {}},
-        'JOB_TYPE_E': {'cost_parameter': '', 'cost_table': {}, 'cost': 1.0},
-        'JOB_TYPE_F': {'cost_parameter': '', 'cost_table': {}, 'foo': None},
-    }
+    costs = [
+        {'job_type': 'JOB_TYPE_A', 'cost_parameter': 'foo', 'cost_table': [{'parameter_value': 'bar', 'cost': 3.0}]},
+        {'job_type': 'JOB_TYPE_B', 'cost': 5.0},
+        {'job_type': 'JOB_TYPE_C', 'cost_parameter': ''},
+        {'job_type': 'JOB_TYPE_D', 'cost_table': {}},
+        {'job_type': 'JOB_TYPE_E', 'cost_parameter': '', 'cost_table': [], 'cost': 1.0},
+        {'job_type': 'JOB_TYPE_F', 'cost_parameter': '', 'cost_table': [], 'foo': None},
+    ]
 
     assert dynamo.jobs._get_credit_cost({'job_type': 'JOB_TYPE_A', 'job_parameters': {'foo': 'bar'}}, costs) == 3.0
     assert dynamo.jobs._get_credit_cost({'job_type': 'JOB_TYPE_B'}, costs) == 5.0
@@ -281,11 +292,11 @@ def test_put_jobs_default_params(tables):
         'JOB_TYPE_B': {'b1': 'b1_default'},
         'JOB_TYPE_C': {},
     }
-    costs = {
-        'JOB_TYPE_A': {'cost': Decimal('1.0')},
-        'JOB_TYPE_B': {'cost': Decimal('1.0')},
-        'JOB_TYPE_C': {'cost': Decimal('1.0')},
-    }
+    costs = [
+        {'job_type': 'JOB_TYPE_A', 'cost': Decimal('1.0')},
+        {'job_type': 'JOB_TYPE_B', 'cost': Decimal('1.0')},
+        {'job_type': 'JOB_TYPE_C', 'cost': Decimal('1.0')},
+    ]
     payload = [
         {},
         {'job_type': 'JOB_TYPE_A'},
@@ -323,24 +334,44 @@ def test_put_jobs_default_params(tables):
 def test_put_jobs_costs(tables):
     tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': Decimal(100)})
 
-    costs = {
-        'RTC_GAMMA': {
+    costs = [
+        {
+            'job_type': 'RTC_GAMMA',
             'cost_parameter': 'resolution',
-            'cost_table': {
-                30: Decimal('5.0'),
-                20: Decimal('15.0'),
-                10: Decimal('60.0'),
-            },
-        },
-        'INSAR_ISCE_BURST': {
+            'cost_table': [
+                {
+                    'parameter_value': 30.0,
+                    'cost': Decimal('5.0'),
+                },
+                {
+                    'parameter_value': 20.0,
+                    'cost': Decimal('15.0'),
+                },
+                {
+                    'parameter_value': 10.0,
+                    'cost': Decimal('60.0'),
+                },
+            ],
+        },
+        {
+            'job_type': 'INSAR_ISCE_BURST',
             'cost_parameter': 'looks',
-            'cost_table': {
-                '20x4': Decimal('0.4'),
-                '10x2': Decimal('0.7'),
-                '5x1': Decimal('1.8'),
-            },
+            'cost_table': [
+                {
+                    'parameter_value': '20x4',
+                    'cost': Decimal('0.4'),
+                },
+                {
+                    'parameter_value': '10x2',
+                    'cost': Decimal('0.7'),
+                },
+                {
+                    'parameter_value': '5x1',
+                    'cost': Decimal('1.8'),
+                },
+            ],
         },
-    }
+    ]
     default_params = {
         'RTC_GAMMA': {'resolution': 30},
         'INSAR_ISCE_BURST': {'looks': '20x4'},