[uss_qualifier/scenarios/netrid/dss/isa_subscription_interactions] sp…

…ecify some check severity in test scenario documentation only (contrib #404)
interuss · Jan 14, 2025 · 6bfaaf3 · 6bfaaf3
1 parent 9ce3338
commit 6bfaaf3
Show file tree

Hide file tree

Showing 14 changed files with 72 additions and 88 deletions.
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py
@@ -141,7 +141,6 @@ def _new_subscription_in_isa_step(self):
             ]:
                 check.record_failed(
                     summary="Subscription response does not include the freshly created ISA",
-                    severity=Severity.High,
                     details=f"The subscription created for the area {self._isa_area} is expected to contain the ISA created for this same area. The returned subscription did not mention it.",
                     query_timestamps=[
                         created_isa.dss_query.query.request.timestamp,
@@ -156,7 +155,6 @@ def _new_subscription_in_isa_step(self):
             if created_subscription.subscription.notification_index != 0:
                 check.record_failed(
                     summary="Subscription notification_index is not 0",
-                    severity=Severity.High,
                     details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 0. The returned subscription has a notification_index of {created_subscription.subscription.notification_index}.",
                     query_timestamps=[created_subscription.query.request.timestamp],
                 )
@@ -192,7 +190,6 @@ def _new_subscription_in_isa_step(self):
             if created_subscription.subscription.id not in subs_to_mutated_isa.keys():
                 check.record_failed(
                     summary="ISA mutation response does not contain expected subscription ID",
-                    severity=Severity.High,
                     details="Mutating an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.",
                     query_timestamps=[
                         created_isa.dss_query.query.request.timestamp,
@@ -213,7 +210,6 @@ def _new_subscription_in_isa_step(self):
                 if sub_to_mutated_isa.notification_index <= 0:
                     check.record_failed(
                         summary="Subscription notification_index has not been increased",
-                        severity=Severity.High,
                         details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 1 or more. The returned subscription has a notification_index of {subs_to_mutated_isa[created_subscription.subscription.id].notification_index}.",
                         query_timestamps=[created_subscription.query.request.timestamp],
                     )
@@ -247,7 +243,6 @@ def _new_subscription_in_isa_step(self):
             if created_subscription.subscription.id not in subs_to_deleted_isa:
                 check.record_failed(
                     summary="ISA deletion response does not contain expected subscription ID",
-                    severity=Severity.High,
                     details="Deleting an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.",
                     query_timestamps=[
                         created_isa.dss_query.query.request.timestamp,
@@ -270,8 +265,7 @@ def _new_subscription_in_isa_step(self):
                 if not notification.success:
                     check.record_failed(
                         "Could not notify ISA subscriber",
-                        Severity.Medium,
-                        f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}",
+                        details=f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}",
                         query_timestamps=[notification.query.request.timestamp],
                     )
 
@@ -289,7 +283,6 @@ def _new_subscription_in_isa_step(self):
                 ):
                     check.record_failed(
                         summary="Subscription notification_index has not been incremented",
-                        severity=Severity.High,
                         details=f"The subscription created for the area {self._isa_area} is expected to have its notification increased after the subscription was deleted."
                         f"The returned subscription has a notification_index of {subs_after_deletion.notification_index}, whilte the previous notification_index for that subscription was {sub_to_mutated_isa.notification_index}",
                         query_timestamps=[created_subscription.query.request.timestamp],
@@ -357,7 +350,6 @@ def _mutate_subscription_towards_isa_boundary_step(self):
             if self._isa_id not in [isa.id for isa in mutated_subscription.isas]:
                 check.record_failed(
                     summary="Subscription response does not include the freshly created ISA",
-                    severity=Severity.High,
                     details=f"The subscription created for the area {self._isa_area} is expected to contain the ISA created for this same area. The returned subscription did not mention it.",
                     query_timestamps=[
                         created_isa.dss_query.query.request.timestamp,
@@ -372,7 +364,6 @@ def _mutate_subscription_towards_isa_boundary_step(self):
             if created_subscription.subscription.notification_index != 0:
                 check.record_failed(
                     summary="Subscription notification_index is not 0",
-                    severity=Severity.High,
                     details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 0. The returned subscription has a notification_index of {created_subscription.subscription.notification_index}.",
                     query_timestamps=[created_subscription.query.request.timestamp],
                 )
@@ -408,7 +399,6 @@ def _mutate_subscription_towards_isa_boundary_step(self):
             if created_subscription.subscription.id not in subs_to_mutated_isa.keys():
                 check.record_failed(
                     summary="ISA mutation response does not contain expected subscription ID",
-                    severity=Severity.High,
                     details="Mutating an ISA to which a subscription was made and then subsequently moved to the ISA's boundary,"
                     " the DSS failed to return the subscription ID in the response.",
                     query_timestamps=[
@@ -431,7 +421,6 @@ def _mutate_subscription_towards_isa_boundary_step(self):
                 if sub_to_mutated_isa.notification_index <= 0:
                     check.record_failed(
                         summary="Subscription notification_index has not been increased",
-                        severity=Severity.High,
                         details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 1 or more. The returned subscription has a notification_index of {subs_to_mutated_isa[created_subscription.subscription.id].notification_index}.",
                         query_timestamps=[created_subscription.query.request.timestamp],
                     )
@@ -465,7 +454,6 @@ def _mutate_subscription_towards_isa_boundary_step(self):
             if created_subscription.subscription.id not in subs_to_deleted_isa:
                 check.record_failed(
                     summary="ISA deletion response does not contain expected subscription ID",
-                    severity=Severity.High,
                     details="Deleting an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.",
                     query_timestamps=[
                         created_isa.dss_query.query.request.timestamp,
@@ -486,8 +474,7 @@ def _mutate_subscription_towards_isa_boundary_step(self):
                     if not notification.success:
                         check.record_failed(
                             "Could not notify ISA subscriber",
-                            Severity.Medium,
-                            f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}",
+                            details=f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}",
                             query_timestamps=[notification.query.request.timestamp],
                         )
 
@@ -505,7 +492,6 @@ def _mutate_subscription_towards_isa_boundary_step(self):
                 ):
                     check.record_failed(
                         summary="Subscription notification_index has not been incremented",
-                        severity=Severity.High,
                         details=f"The subscription created for the area {self._isa_area} is expected to have its notification increased after the subscription was deleted."
                         f"The returned subscription has a notification_index of {subs_after_deletion.notification_index}, whilte the previous notification_index for that subscription was {sub_to_mutated_isa.notification_index}",
                         query_timestamps=[created_subscription.query.request.timestamp],

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py
@@ -306,7 +306,6 @@ def put_isa_expect_response_code(
             q=mutated_isa.dss_query,
             fail_msg="ISA Put succeeded when expecting a failure",
             required_status_code=expected_error_codes,
-            severity=Severity.High,
             fail_details=f"The submitted query was expected to fail. Payload: {mutated_isa.dss_query.query.request.json}",
         )
         return mutated_isa
@@ -890,7 +889,6 @@ def del_sub(
             if sub_version != del_sub.subscription.version:
                 check.record_failed(
                     summary=f"Deleted subscription did not match",
-                    severity=Severity.High,
                     details=f"DSS reported deletion of version {sub_version} while expecting {del_sub.subscription.version}",
                     query_timestamps=[del_sub.query.request.timestamp],
                 )

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md
@@ -30,7 +30,7 @@ This test case creates an ISA with a short lifetime and verifies that it is not
 
 ### ISA Expiry test step
 
-#### Create short-lived ISA check
+#### 🛑 Create short-lived ISA check
 
 Not allowing an ISA to be created violates **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**
 

diff --git a/...ng/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md b/...ng/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md
@@ -47,61 +47,61 @@ This test case will do the following, using the DSS being tested:
 This test step checks for interactions between ISAs and a subscription that is created within the ISA, then
 subsequently mutated to only barely intersect with the ISA.
 
-#### Create an ISA check
+#### 🛑 Create an ISA check
 
 If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly.
 
-#### Create a subscription within the ISA footprint check
+#### 🛑 Create a subscription within the ISA footprint check
 
 The DSS should allow the creation of a subscription within the ISA footprint, otherwise it is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**
 
-#### Subscription for the ISA's area mentions the ISA check
+#### 🛑 Subscription for the ISA's area mentions the ISA check
 
 A subscription that is created for a volume that intersects with the previously created ISA should mention
 the previously created ISA. If not, the serving DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Newly created subscription has a notification_index of 0 check
+#### 🛑 Newly created subscription has a notification_index of 0 check
 
 A newly created subscription is expected to have a notification index of 0, otherwise the DSS implementation under
 test does not comply with **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**
 
-#### Mutate the ISA check
+#### 🛑 Mutate the ISA check
 
 If the ISA cannot be mutated, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly.
 
-#### Response to the mutation of the ISA contains subscription ID check
+#### 🛑 Response to the mutation of the ISA contains subscription ID check
 
 When an ISA is mutated, the DSS must return the identifiers for any subscription that was made to the ISA,
 or be in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Subscription to an ISA has its notification index incremented after mutation check
+#### 🛑 Subscription to an ISA has its notification index incremented after mutation check
 
 When an ISA is mutated, the DSS must increment the notification index of any subscription to that ISA,
 and return the up-to-date subscription in the response to the query mutating the ISA.
 
 Failure to do so means that the DSS is not properly implementing **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Delete the ISA check
+#### 🛑 Delete the ISA check
 
 If that ISA cannot be deleted, the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met.
 
-#### Response to the deletion of the ISA contains subscription ID check
+#### 🛑 Response to the deletion of the ISA contains subscription ID check
 
 When an ISA is deleted, the DSS must return the identifiers for any subscription that was made to the ISA,
 or be in violation of **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Subscription to an ISA has its notification index incremented after deletion check
+#### 🛑 Subscription to an ISA has its notification index incremented after deletion check
 
 When an ISA is deleted, the DSS must increment the notification index of any subscription to that ISA,
 and return the up-to-date subscription in the response to the query deleting the ISA.
 
 Failure to do so means that the DSS is not properly implementing **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Subscription can be deleted check
+#### 🛑 Subscription can be deleted check
 
 **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created.
 
-#### Notified subscriber check
+#### ⚠️ Notified subscriber check
 
 Notifications to any subscriber to the created ISA need to be successful.  If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met.
 
@@ -110,38 +110,38 @@ Notifications to any subscriber to the created ISA need to be successful.  If a
 This test step checks for interactions between ISAs and a subscription that is created within the ISA and the mutated
 to only barely overlap with the ISA.
 
-#### Create an ISA check
+#### 🛑 Create an ISA check
 
 If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly.
 
-#### Create a subscription within the ISA footprint check
+#### 🛑 Create a subscription within the ISA footprint check
 
 The DSS should allow the creation of a subscription within the ISA footprint, otherwise it is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**
 
-#### Mutate the subscription towards the ISA boundary check
+#### 🛑 Mutate the subscription towards the ISA boundary check
 
 The DSS should allow a valid mutation of a subscription's area, otherwise it is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**
 
-#### Subscription for the ISA's area mentions the ISA check
+#### 🛑 Subscription for the ISA's area mentions the ISA check
 
 A subscription that is created for a volume that intersects with the previously created ISA should mention
 the previously created ISA. If not, the serving DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Mutated subscription has a notification_index of 0 check
+#### 🛑 Mutated subscription has a notification_index of 0 check
 
 A newly created subscription is expected to have a notification index of 0, otherwise the DSS implementation under
 test does not comply with **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**
 
-#### Mutate the ISA check
+#### 🛑 Mutate the ISA check
 
 If the ISA cannot be mutated, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly.
 
-#### Response to the mutation of the ISA contains subscription ID check
+#### 🛑 Response to the mutation of the ISA contains subscription ID check
 
 When an ISA is mutated, the DSS must return the identifiers for any subscription that was made to the ISA,
 or be in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Subscription to an ISA has its notification index incremented after mutation check
+#### 🛑 Subscription to an ISA has its notification index incremented after mutation check
 
 When an ISA is mutated, the DSS must increment the notification index of any subscription to that ISA,
 and return the up-to-date subscription in the response to the query mutating the ISA.
@@ -153,27 +153,27 @@ Failure to do so means that the DSS is not properly implementing **[astm.f3411.v
 A subscription that is created for a volume that only barely overlaps with the previously created ISA should still
 contain the ISA in the reply from the server, otherwise the DSS does not comply with **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**
 
-#### Delete the ISA check
+#### 🛑 Delete the ISA check
 
 If that ISA cannot be deleted, the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met.
 
-#### Response to the deletion of the ISA contains subscription ID check
+#### 🛑 Response to the deletion of the ISA contains subscription ID check
 
 When an ISA is deleted, the DSS must return the identifiers for any subscription that was made to the ISA,
 or be in violation of **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Subscription to an ISA has its notification index incremented after deletion check
+#### 🛑 Subscription to an ISA has its notification index incremented after deletion check
 
 When an ISA is deleted, the DSS must increment the notification index of any subscription to that ISA,
 and return the up-to-date subscription in the response to the query deleting the ISA.
 
 Failure to do so means that the DSS is not properly implementing **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**.
 
-#### Subscription can be deleted check
+#### 🛑 Subscription can be deleted check
 
 **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created.
 
-#### Notified subscriber check
+#### ⚠️ Notified subscriber check
 
 Notifications to any subscriber to the created ISA need to be successful.  If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met.