feat: Add schema validation tests (#5654)

* Add validation tests for JSON schema

* Update schema logic for space separated list type

* Fix failing Windows tests

* Actually fix failing tests

* Implement requested changes

* Self dot fail

---------

Co-authored-by: Leonardo Gama <[email protected]>

samcli/commands/_utils/options.py

@@ -820,7 +820,7 @@ def _space_separated_list_func_type(value):
     raise ValueError()
 
 
-_space_separated_list_func_type.__name__ = "LIST"
+_space_separated_list_func_type.__name__ = "list,string"
 
 
 def generate_next_command_recommendation(command_tuples: List[Tuple[str, str]]) -> str:

schema/samcli.json

@@ -1085,7 +1085,10 @@
                 },
                 "notification_arns": {
                   "title": "notification_arns",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "ARNs of SNS topics that AWS Cloudformation associates with the stack.",
                   "items": {
                     "type": "string"
@@ -1122,7 +1125,10 @@
                 },
                 "capabilities": {
                   "title": "capabilities",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "List of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html",
                   "items": {
                     "type": "string"
@@ -1560,7 +1566,10 @@
                 },
                 "notification_arns": {
                   "title": "notification_arns",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "ARNs of SNS topics that AWS Cloudformation associates with the stack.",
                   "items": {
                     "type": "string"
@@ -1576,7 +1585,10 @@
                 },
                 "capabilities": {
                   "title": "capabilities",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "List of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html",
                   "default": [
                     "CAPABILITY_NAMED_IAM",

tests/unit/schema/test_schema_validation.py

@@ -0,0 +1,53 @@
+import os
+from pathlib import Path
+import jsonschema
+from parameterized import parameterized
+from unittest import TestCase
+
+from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER
+from schema.make_schema import generate_schema
+
+
+class TestSchemaValidation(TestCase):
+    schema = None
+    testdata_dir = None
+
+    @classmethod
+    def setUpClass(cls):
+        cls.schema = generate_schema()
+        testing_dir = Path(__name__).resolve().parents[0]
+        cls.testdata_dir = str(Path(testing_dir, "tests", "unit", "schema", "testdata"))
+
+    def test_samconfig_validates_against_schema(self):
+        self.assertIsNotNone(self.schema, "Schema was not set")
+
+        passing_tests_dir = Path(self.testdata_dir, "passing_tests")
+
+        # Read in and assert all files in passing_tests pass
+        for config_file_path in os.listdir(passing_tests_dir):
+            config_file = FILE_MANAGER_MAPPER[Path(config_file_path).suffix].read(
+                Path(str(passing_tests_dir), config_file_path)
+            )
+            self.assertNotEqual(config_file, {}, f"Config file {config_file_path} should be read correctly")
+
+            try:
+                jsonschema.validate(config_file, self.schema)
+            except jsonschema.ValidationError as e:
+                self.fail(f"File {config_file_path} not validating: {e.message}")
+
+    def test_samconfig_doesnt_validate_against_schema(self):
+        self.assertIsNotNone(self.schema, "Schema was not set")
+
+        failing_tests_dir = Path(self.testdata_dir, "failing_tests")
+
+        # Read in and assert all files in failing_tests fail
+        for config_file_path in os.listdir(failing_tests_dir):
+            config_file = FILE_MANAGER_MAPPER[Path(config_file_path).suffix].read(
+                Path(str(failing_tests_dir), config_file_path)
+            )
+            self.assertNotEqual(config_file, {}, f"Config file {config_file_path} should be read correctly")
+
+            with self.assertRaises(
+                jsonschema.ValidationError, msg=f"Config file {config_file_path} should not validate against schema"
+            ):
+                jsonschema.validate(config_file, self.schema)

tests/unit/schema/testdata/failing_tests/buildcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.build]
+debug = false

tests/unit/schema/testdata/failing_tests/buildcmd_noparams.yaml

@@ -0,0 +1,4 @@
+version: 0.1
+default:
+  build:
+    debug: false

tests/unit/schema/testdata/failing_tests/buildcmd_wrongtype_build_dir.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.build.parameters]
+build_dir = false

tests/unit/schema/testdata/failing_tests/buildcmd_wrongtype_build_dir.yaml

@@ -0,0 +1,6 @@
+version: 0.1
+
+default:
+  build:
+    parameters:
+      build_dir: false

tests/unit/schema/testdata/failing_tests/buildcmd_wrongtype_debug.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.build.parameters]
+debug = "false"

tests/unit/schema/testdata/failing_tests/buildcmd_wrongtype_debug.yaml

@@ -0,0 +1,6 @@
+version: 0.1
+
+default:
+  build:
+    parameters:
+      debug: "False"

tests/unit/schema/testdata/failing_tests/buildcmd_wrongtype_parameter_overrides.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.build.parameters]
+parameter_overrides = 10

tests/unit/schema/testdata/failing_tests/buildcmd_wrongtype_parameter_overrides.yaml

@@ -0,0 +1,6 @@
+version: 0.1
+
+default:
+  build:
+    parameters:
+      parameter_overrides: 10

tests/unit/schema/testdata/failing_tests/deletecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.delete]
+debug = false

tests/unit/schema/testdata/failing_tests/deploycmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.deploy]
+debug = false

tests/unit/schema/testdata/failing_tests/initcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.init]
+debug = false

tests/unit/schema/testdata/failing_tests/list_endpointscmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.list_endpoints]
+debug = false

tests/unit/schema/testdata/failing_tests/list_resourcescmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.list_resources]
+debug = false

tests/unit/schema/testdata/failing_tests/list_stack_outputscmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.list_stack_outputs]
+debug = false

tests/unit/schema/testdata/failing_tests/local_invokecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_invoke]
+debug = false

tests/unit/schema/testdata/failing_tests/local_invokecmd_wrongtype_debug_port.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_invoke.parameters]
+debug_port = "10"

tests/unit/schema/testdata/failing_tests/local_invokecmd_wrongtype_debug_port.yaml

@@ -0,0 +1,6 @@
+version: 0.1
+
+default:
+  local_invoke:
+    parameters:
+      debug_port: "10"

tests/unit/schema/testdata/failing_tests/local_start_apicmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_start_api]
+debug = false

tests/unit/schema/testdata/failing_tests/local_start_lambdacmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_start_lambda]
+debug = false

tests/unit/schema/testdata/failing_tests/logscmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.logs]
+debug = false

tests/unit/schema/testdata/failing_tests/noversion.toml

@@ -0,0 +1,26 @@
+# version = 0.1
+
+[default]
+[default.global]
+[default.global.parameters]
+stack_name = "sam-app"
+
+[default.build.parameters]
+cached = true
+parallel = true
+
+[default.deploy.parameters]
+capabilities = "CAPABILITY_IAM"
+confirm_changeset = true
+resolve_s3 = true
+
+[default.sync.parameters]
+watch = true
+
+[default.local_start_api.parameters]
+warm_containers = "EAGER"
+
+[prod]
+[prod.sync]
+[prod.sync.parameters]
+watch = false

tests/unit/schema/testdata/failing_tests/noversion.yaml

@@ -0,0 +1,4 @@
+default:
+  build:
+    parameters:
+      debug: false

tests/unit/schema/testdata/failing_tests/packagecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.package]
+debug = false

tests/unit/schema/testdata/failing_tests/packagecmd_wrongtype_image_repositories.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.package.parameters]
+image_repositories = "Image repositories"

tests/unit/schema/testdata/failing_tests/packagecmd_wrongtype_image_repositories.yaml

@@ -0,0 +1,6 @@
+version: 0.1
+
+default:
+  package:
+    parameters:
+      image_repositories: Image repositories

tests/unit/schema/testdata/failing_tests/pipeline_bootstrapcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.pipeline_bootstrap]
+debug = false

tests/unit/schema/testdata/failing_tests/pipeline_initcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.pipeline_init]
+debug = false

tests/unit/schema/testdata/failing_tests/publishcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.publish]
+debug = false

tests/unit/schema/testdata/failing_tests/remote_invokecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.remote_invoke]
+debug = false

tests/unit/schema/testdata/failing_tests/synccmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.sync]
+debug = false

tests/unit/schema/testdata/failing_tests/tracescmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.traces]
+debug = false

tests/unit/schema/testdata/failing_tests/validatecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.validate]
+debug = false

tests/unit/schema/testdata/passing_tests/buildcmd.toml

@@ -0,0 +1,36 @@
+version = 0.1
+
+[default.build.parameters]
+hook_name = "Hook name"
+skip_prepare_infra = false
+use_container = false
+container_env_var = "Container env var"
+container_env_var_file = "Container env var file"
+build_image = "Build image"
+exclude = "Exclude"
+parallel = false
+mount_with = "READ"
+build_dir = "Build dir"
+cache_dir = "Cache dir"
+base_dir = "Base dir"
+manifest = "Manifest"
+cached = false
+template_file = "Template file"
+parameter_overrides = "Parameter overrides"
+skip_pull_image = false
+docker_network = "Docker network"
+beta_features = false
+debug = false
+profile = "Profile"
+region = "Region"
+
+[other.build.parameters]
+skip_prepare_infra = true
+use_container = true
+parallel = true
+mount_with = "WRITE"
+cached = true
+parameter_overrides = ["Parameter", "Overrides"]
+skip_pull_image = true
+beta_features = true
+debug = true

tests/unit/schema/testdata/passing_tests/deletecmd.toml

@@ -0,0 +1,16 @@
+version = 0.1
+
+[default.delete.parameters]
+stack_name = "Stack name"
+no_prompts = false
+s3_bucket = "S3 Bucket"
+s3_prefix = "S3 Prefix"
+profile = "Profile"
+region = "Region"
+beta_features = false
+debug = false
+
+[other.delete.parameters]
+no_prompts = true
+beta_features = true
+debug = true