feat: JSON schema support for samconfig files

.github/workflows/build.yml

@@ -25,13 +25,15 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - make-pr
+      - validate-schema
       - integration-tests
       - smoke-and-functional-tests
       - docker-disabled
     steps:
       - name: report-failure
         if : |
           needs.make-pr.result != 'success' ||
+          needs.validate-schema.result != 'success' ||
           needs.integration-tests.result != 'success' ||
           needs.smoke-and-functional-tests.result != 'success' ||
           needs.docker-disabled.result != 'success'
@@ -64,6 +66,27 @@ jobs:
     - run: make init
     - run: make pr
 
+  validate-schema:
+    name: Validate JSON schema
+    if: github.repository_owner == 'aws'
+    permissions:
+      pull-requests: write
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        name: Install Python 3.11
+        with:
+          python-version: 3.11
+      - run: make init
+      - run: |
+          diff <( cat schema/samcli.json ) <( python schema/make_schema.py; cat schema/samcli.json ) && exit 0 # exit if schema is unchanged
+          echo "The generated schema differs from that in the PR. Please run 'make schema'."
+          exit 1
+        name: Generate and compare the schema
+        shell: bash
+
   integration-tests:
     name: Integration Tests / ${{ matrix.os }} / ${{ matrix.python }} / ${{ matrix.tests_folder }}
     if: github.repository_owner == 'aws'

Makefile

@@ -2,17 +2,19 @@
 # environment variable.
 SAM_CLI_TELEMETRY ?= 0
 
+.PHONY: schema
+
 init:
 	SAM_CLI_DEV=1 pip install -e '.[dev]'
 
 test:
 	# Run unit tests
 	# Fail if coverage falls below 95%
-	pytest --cov samcli --cov-report term-missing --cov-fail-under 94 tests/unit
+	pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit
 
 test-cov-report:
 	# Run unit tests with html coverage report
-	pytest --cov samcli --cov-report html --cov-fail-under 94 tests/unit
+	pytest --cov samcli --cov schema --cov-report html --cov-fail-under 94 tests/unit
 
 integ-test:
 	# Integration tests don't need code coverage
@@ -34,24 +36,27 @@ smoke-test:
 
 lint:
 	# Linter performs static analysis to catch latent bugs
-	ruff samcli
+	ruff samcli schema
 	# mypy performs type check
-	mypy --exclude /testdata/ --exclude /init/templates/ --no-incremental setup.py samcli tests
+	mypy --exclude /testdata/ --exclude /init/templates/ --no-incremental setup.py samcli tests schema
 
 # Command to run everytime you make changes to verify everything works
 dev: lint test
 
 black:
-	black setup.py samcli tests
+	black setup.py samcli tests schema
 
 black-check:
-	black --check setup.py samcli tests
+	black --check setup.py samcli tests schema
 
 format: black
 	ruff samcli --fix
 
+schema: 
+	python schema/make_schema.py
+
 # Verifications to run before sending a pull request
-pr: init dev black-check
+pr: init dev schema black-check
 
 # (jfuss) We updated to have two requirement files, one for mac and one for linux. This
 # is meant to be a short term fix when upgrading the Linux installer to be python3.11 from 

samcli/cli/types.py

@@ -85,8 +85,7 @@ class CfnParameterOverridesType(click.ParamType):
 
     ordered_pattern_match = [_pattern_1, _pattern_2]
 
-    # NOTE(TheSriram): name needs to be added to click.ParamType requires it.
-    name = ""
+    name = "string,list"
 
     def convert(self, value, param, ctx):
         result = {}
@@ -140,8 +139,7 @@ class CfnMetadataType(click.ParamType):
 
     _pattern = r"(?:{key}={value})".format(key=PARAM_AND_METADATA_KEY_REGEX, value=VALUE_REGEX_COMMA_DELIM)
 
-    # NOTE(TheSriram): name needs to be added to click.ParamType requires it.
-    name = ""
+    name = "string"
 
     def convert(self, value, param, ctx):
         result = {}
@@ -196,8 +194,7 @@ def __init__(self, multiple_values_per_key=False):
 
     _pattern = r"{tag}={tag}".format(tag=_generate_match_regex(match_pattern=TAG_REGEX, delim=" "))
 
-    # NOTE(TheSriram): name needs to be added to click.ParamType requires it.
-    name = ""
+    name = "list"
 
     def convert(self, value, param, ctx):
         result = {}
@@ -301,8 +298,7 @@ class SigningProfilesOptionType(click.ParamType):
 
     pattern = r"(?:(?: )([A-Za-z0-9\"]+)=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
 
-    # Note: this is required, otherwise it is failing when running help
-    name = ""
+    name = "string"
 
     def convert(self, value, param, ctx):
         """
@@ -393,7 +389,7 @@ def transform(self, *args, **kwargs):
 
     # Transformation callback function checks if the received option value is a valid ECR url.
     transformer = Transformer(converter=click.STRING, transformation=is_ecr_url)
-    name = ""
+    name = "string"
 
     def convert(self, value, param, ctx):
         """
@@ -410,7 +406,7 @@ class ImageRepositoriesType(click.ParamType):
     Custom Parameter Type for Multi valued Image Repositories option.
     """
 
-    name = ""
+    name = "list"
     KEY_VALUE_PAIR_LENGTH = 2
 
     def convert(self, value, param, ctx):
@@ -431,7 +427,7 @@ class RemoteInvokeBotoApiParameterType(click.ParamType):
     Custom Parameter Type for Multi valued Boto API parameter option of remote invoke command.
     """
 
-    name = ""
+    name = "list"
     MIN_KEY_VALUE_PAIR_LENGTH = 2
 
     def convert(self, value, param, ctx):

samcli/commands/_utils/options.py

@@ -367,13 +367,6 @@ def common_observability_click_options():
             help="Fetch events up to this time. Time can be relative values like '5mins ago', 'tomorrow' or "
             "formatted timestamp like '2018-01-01 10:10:10'",
         ),
-        click.option(
-            "--tail",
-            "-t",
-            is_flag=True,
-            help="Tail events. This will ignore the end time argument and continue to fetch events as they "
-            "become available. If option --tail without a --name will pull from all possible resources",
-        ),
         click.option(
             "--output",
             help="""
@@ -827,7 +820,7 @@ def _space_separated_list_func_type(value):
     raise ValueError()
 
 
-_space_separated_list_func_type.__name__ = "LIST"
+_space_separated_list_func_type.__name__ = "list,string"
 
 
 def generate_next_command_recommendation(command_tuples: List[Tuple[str, str]]) -> str:

samcli/commands/build/command.py

@@ -83,7 +83,7 @@
     multiple=True,  # Can pass in multiple env vars
     required=False,
     help="Environment variables to be passed into build containers"
-    "Resource format (FuncName.VarName=Value) or Global format (VarName=Value)."
+    "\nResource format (FuncName.VarName=Value) or Global format (VarName=Value)."
     "\n\n Example: --container-env-var Func1.VAR1=value1 --container-env-var VAR2=value2",
     cls=ContainerOptions,
 )

samcli/commands/init/command.py

@@ -106,6 +106,7 @@ def wrapped(*args, **kwargs):
 
 @click.command(
     "init",
+    help=HELP_TEXT,
     short_help=HELP_TEXT,
     context_settings={"max_content_width": 120},
     cls=InitCommand,

samcli/commands/local/cli_common/options.py

@@ -231,7 +231,7 @@ def warm_containers_common_options(f):
         click.option(
             "--debug-function",
             help="Optional. Specifies the Lambda Function logicalId to apply debug options to when"
-            " --warm-containers is specified.  This parameter applies to --debug-port, --debugger-path,"
+            " --warm-containers is specified. This parameter applies to --debug-port, --debugger-path,"
             " and --debug-args.",
             type=click.STRING,
             multiple=False,

samcli/commands/logs/command.py

@@ -71,6 +71,14 @@
     "supported by AWS CloudWatch Logs. See the AWS CloudWatch Logs documentation for the syntax "
     "https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html",
 )
+@click.option(
+    "--tail",
+    "-t",
+    is_flag=True,
+    help="Tail events. This will ignore the end time argument and continue to fetch events as they "
+    "become available. If option --tail is provided without a --name, one will be pulled from all "
+    "possible resources",
+)
 @click.option(
     "--include-traces",
     "-i",

samcli/commands/traces/command.py

@@ -35,6 +35,13 @@
     multiple=True,
     help="Fetch specific trace by providing its id",
 )
+@click.option(
+    "--tail",
+    "-t",
+    is_flag=True,
+    help="Tail events. This will ignore the end time argument and continue to fetch events as they "
+    "become available.",
+)
 @common_observability_options
 @cli_framework_options
 @aws_creds_options
@@ -46,20 +53,20 @@
 def cli(
     ctx,
     trace_id,
+    tail,
     start_time,
     end_time,
-    tail,
     output,
     config_file,
     config_env,
 ):
     """
     `sam traces` command entry point
     """
-    do_cli(trace_id, start_time, end_time, tail, output, ctx.region)
+    do_cli(trace_id, tail, start_time, end_time, output, ctx.region)
 
 
-def do_cli(trace_ids, start_time, end_time, tailing, output, region):
+def do_cli(trace_ids, tailing, start_time, end_time, output, region):
     """
     Implementation of the ``cli`` method
     """

samcli/lib/config/samconfig.py

@@ -87,7 +87,7 @@ def get_all(self, cmd_names, section, env=DEFAULT_ENV):
         self.document = self._read()
 
         config_content = self.document.get(env, {})
-        params = config_content.get(self._to_key(cmd_names), {}).get(section, {})
+        params = config_content.get(self.to_key(cmd_names), {}).get(section, {})
         if DEFAULT_GLOBAL_CMDNAME in config_content:
             global_params = config_content.get(DEFAULT_GLOBAL_CMDNAME, {}).get(section, {})
             global_params.update(params.copy())
@@ -116,7 +116,7 @@ def put(self, cmd_names, section, key, value, env=DEFAULT_ENV):
         # Empty document prepare the initial structure.
         # self.document is a nested dict, we need to check each layer and add new tables, otherwise duplicated key
         # in parent layer will override the whole child layer
-        cmd_name_key = self._to_key(cmd_names)
+        cmd_name_key = self.to_key(cmd_names)
         env_content = self.document.get(env, {})
         cmd_content = env_content.get(cmd_name_key, {})
         param_content = cmd_content.get(section, {})
@@ -284,6 +284,6 @@ def _version_sanity_check(version: Any) -> None:
             raise SamConfigVersionException(f"'{VERSION_KEY}' key is not present or is in unrecognized format. ")
 
     @staticmethod
-    def _to_key(cmd_names: Iterable[str]) -> str:
+    def to_key(cmd_names: Iterable[str]) -> str:
         # construct a parsed name that is of the format: a_b_c_d
         return "_".join([cmd.replace("-", "_").replace(" ", "_") for cmd in cmd_names])

samcli/local/common/runtime_template.py

@@ -83,13 +83,15 @@ def get_local_lambda_images_location(mapping, runtime):
     return os.path.join(_lambda_images_templates, runtime, dir_name + "-lambda-image")
 
 
-SUPPORTED_DEP_MANAGERS: List[str] = list(
-    set(
-        {
-            c.get("dependency_manager")  # type: ignore
-            for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))
-            if c.get("dependency_manager")
-        }
+SUPPORTED_DEP_MANAGERS: List[str] = sorted(
+    list(
+        set(
+            {
+                c.get("dependency_manager")  # type: ignore
+                for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))
+                if c.get("dependency_manager")
+            }
+        )
     )
 )
 
@@ -145,7 +147,7 @@ def get_local_lambda_images_location(mapping, runtime):
     "ruby2.7": "amazon/ruby2.7-base",
 }
 
-LAMBDA_IMAGES_RUNTIMES: List = list(set(LAMBDA_IMAGES_RUNTIMES_MAP.values()))
+LAMBDA_IMAGES_RUNTIMES: List = sorted(list(set(LAMBDA_IMAGES_RUNTIMES_MAP.values())))
 
 # Schemas Code lang is a MINIMUM supported version
 # - this is why later Lambda runtimes can be mapped to earlier Schemas Code Languages

schema/exceptions.py

@@ -0,0 +1,5 @@
+"""Exceptions related to schema generation."""
+
+
+class SchemaGenerationException(Exception):
+    pass