Feat/CodeCommit provider - set "default-scm-codecommit-account-id" in adfconfig.yml and set default fallback

docs/admin-guide.md

@@ -213,6 +213,13 @@ Config has five components in `main-notification-endpoint`, `scp`, `scm`,
     `main`. As new repositories will most likely use this branch name as their
     default branch.
 
+  - **default-scm-codecommit-account-id** allows you to configure the default account id that
+    should be used with all source-code management platforms that ADF supports.
+    If not set here, the deployment account id is taken as default value. 
+    The CodeCommit account-id can be still be overwritten with an explicit account id in the individual deployment map.
+    The CodeCommit provider guide provides more details: [providers-guide.md.yml: CodeCommit](./providers-guide.md#codecommit).
+
+
 ## Accounts
 
 ### Management account

docs/providers-guide.md

@@ -63,11 +63,13 @@ Provider type: `codecommit`.
 
 #### Properties
 
-- *account_id* - *(String)* **(required)**
+- *account_id* - *(String)* **(optional)**
   - The AWS Account ID where the Source Repository is located. If the repository
     does not exist it will be created via AWS CloudFormation on the source
     account along with the associated cross account CloudWatch event action to
     trigger the pipeline.
+  - Additionally, the default account id for CodeCommit, can be set in [adfconfig.yml: config/scm/default-scm-codecommit-account-id](./admin-guide.md#adfconfig). 
+  - If not set here in the provider and if not set in adfconfig.yml, the deployment account id will be used as default value.
 - *repository* - *(String)* defaults to name of the pipeline.
   - The AWS CodeCommit repository name.
 - *branch* - *(String)* default to configured [adfconfig.yml: config/scm/default-scm-branch](./admin-guide.md#adfconfig).

src/lambda_codebase/initial_commit/adfconfig.yml.j2

@@ -27,3 +27,6 @@ config:
   scm:
     auto-create-repositories: enabled
     default-scm-branch: main
+    # Optional:
+    # default-scm-codecommit-account-id: "123456789012"
+

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py

@@ -11,10 +11,12 @@
 from rule import Rule
 from logger import configure_logger
 from cloudwatch import ADFMetrics
-
+from errors import ParameterNotFoundError
+from parameter_store import ParameterStore
 
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
+DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 CLOUDWATCH = boto3.client("cloudwatch")
 METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/RULE")
 
@@ -58,6 +60,37 @@ def lambda_handler(event, _):
         .get("properties", {})
         .get("account_id")
     )
+
+    # Resolve source_account_id in case it is not set
+    if source_provider == "codecommit" and not source_account_id:
+        # Evaluate as follows: 
+        # If source_account_id not set, we have to set it as follows:
+        #   - set via default_scm_codecommit_account_id (if exists)
+        #   - or set via ADF_DEPLOYMENT_ACCOUNT_ID
+        deployment_account_id = DEPLOYMENT_ACCOUNT_ID
+        try:
+            parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
+            default_scm_codecommit_account_id = parameter_store.fetch_parameter(
+                "/adf/scm/default-scm-codecommit-account-id"
+            )
+        except ParameterNotFoundError:
+            default_scm_codecommit_account_id = deployment_account_id
+        if not source_account_id:
+            print("account_id not found in source_props - recreate it!")
+            if default_scm_codecommit_account_id:
+                source_account_id = default_scm_codecommit_account_id
+            else:
+                source_account_id = deployment_account_id
+            if "properties" in pipeline["default_providers"]["source"]:
+                # append to properties
+                pipeline["default_providers"]["source"]["properties"]["account_id"] = source_account_id
+            else:
+                # recreate properties
+                source_props =  {
+                    "account_id": source_account_id
+                }
+                pipeline["default_providers"]["source"]["properties"] = source_props
+
     if (
         source_provider == "codecommit"
         and source_account_id

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py

@@ -8,6 +8,7 @@
 from repo import Repo
 
 from logger import configure_logger
+from errors import ParameterNotFoundError
 from cloudwatch import ADFMetrics
 from parameter_store import ParameterStore
 
@@ -16,7 +17,7 @@
 METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/REPO")
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
-
+DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
 
 def lambda_handler(event, _):
     """
@@ -67,6 +68,34 @@ def lambda_handler(event, _):
         .get("properties", {})
         .get("repository", {})
     )
+
+    # Resolve code_account_id in case it is not set
+    # Evaluate as follows: 
+    # If source_account_id not set, we have to set it as follows:
+    #   - set via default_scm_codecommit_account_id (if exists)
+    #   - or set via ADF_DEPLOYMENT_ACCOUNT_ID
+    try:
+        default_scm_codecommit_account_id = parameter_store.fetch_parameter(
+            "/adf/scm/default-scm-codecommit-account-id"
+        )
+    except ParameterNotFoundError:
+        default_scm_codecommit_account_id = DEPLOYMENT_ACCOUNT_ID
+    if not code_account_id:
+        print("account_id not found in source_props - recreate it!")
+        if default_scm_codecommit_account_id:
+            code_account_id = default_scm_codecommit_account_id
+        else:
+            code_account_id = DEPLOYMENT_ACCOUNT_ID
+        if "properties" in pipeline["default_providers"]["source"]:
+            # append to properties
+            pipeline["default_providers"]["source"]["properties"]["account_id"] = code_account_id
+        else:
+            # recreate properties
+            source_props =  {
+                "account_id": code_account_id
+            }
+            pipeline["default_providers"]["source"]["properties"] = source_props
+
     if (
         code_account_id
         and str(code_account_id).isdigit()

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

@@ -72,6 +72,9 @@ def fetch_required_ssm_params(pipeline_input, regions):
             output["default_scm_branch"] = parameter_store.fetch_parameter(
                 "default_scm_branch",
             )
+            output["default_scm_codecommit_account_id"] = parameter_store.fetch_parameter(
+                "/adf/scm/default-scm-codecommit-account-id",
+            )
             codestar_connection_path = (
                 pipeline_input
                 .get("default_providers", {})
@@ -155,6 +158,9 @@ def generate_pipeline_inputs(
     data["pipeline_input"]["default_scm_branch"] = data["ssm_params"].get(
         "default_scm_branch",
     )
+    data["pipeline_input"]["default_scm_codecommit_account_id"] = data["ssm_params"].get(
+        "default_scm_codecommit_account_id",
+    )
     store_regional_parameter_config(
         pipeline_object,
         parameter_store,

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml

@@ -160,6 +160,18 @@ Resources:
                 - "lambda.amazonaws.com"
             Action:
               - "sts:AssumeRole"
+      Policies:
+        - PolicyName: "adf-pipeline-create-update-rule-policy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - "ssm:GetParameter"
+                  - "ssm:GetParameters"
+                  - "ssm:GetParametersByPath"
+                Resource:
+                  - "*"
 
   CreateRepositoryLambdaRole:
     Type: "AWS::IAM::Role"

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py

@@ -151,6 +151,13 @@ def prepare_deployment_account(sts, deployment_account_id, config):
             ADF_DEFAULT_SCM_FALLBACK_BRANCH,
         )
     )
+    deployment_account_parameter_store.put_parameter(
+        '/adf/scm/default-scm-codecommit-account-id',
+        config.config.get('scm', {}).get(
+            'default-scm-codecommit-account-id',
+            deployment_account_id,
+        )
+    )
     auto_create_repositories = config.config.get(
         'scm', {}).get('auto-create-repositories')
     if auto_create_repositories is not None:

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codecommit.py

@@ -16,13 +16,36 @@
 ADF_DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
 ADF_DEFAULT_BUILD_TIMEOUT = 20
 
-
 class CodeCommit(Construct):
     def __init__(self, scope: Construct, id: str, map_params: dict, **kwargs): #pylint: disable=W0622
         super().__init__(scope, id, **kwargs)
         default_providers = map_params.get("default_providers", {})
         source_props = default_providers.get("source", {}).get("properties", {})
-        account_id = source_props.get("account_id", ADF_DEPLOYMENT_ACCOUNT_ID)
+
+        # Resolve account_id in case it is not set
+        # Evaluate as follows: 
+        # If account_id not set, we have to set it as follows:
+        #   - set via default_scm_codecommit_account_id (if exists)
+        #   - or set via ADF_DEPLOYMENT_ACCOUNT_ID
+        default_scm_codecommit_account_id = map_params.get("default_scm_codecommit_account_id", "")
+        if not source_props.get("account_id"):
+            print("account_id not found in source_props - recreate it!")
+            if default_scm_codecommit_account_id:
+                account_id = default_scm_codecommit_account_id
+            else:
+                account_id = ADF_DEPLOYMENT_ACCOUNT_ID
+            if "properties" in map_params["default_providers"]["source"]:
+                # append to properties
+                map_params["default_providers"]["source"]["properties"]["account_id"] = account_id
+            else:
+                # recreate properties
+                source_props =  {
+                    "account_id": account_id
+                }
+                map_params["default_providers"]["source"]["properties"] = source_props
+        else:
+            account_id = source_props.get("account_id", ADF_DEPLOYMENT_ACCOUNT_ID)
+
         self.source = _codepipeline.CfnPipeline.StageDeclarationProperty(
             name=f"Source-{account_id}",
             actions=[

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py

@@ -25,6 +25,7 @@
 ADF_PIPELINE_PREFIX = os.environ.get("ADF_PIPELINE_PREFIX", "")
 ADF_DEFAULT_BUILD_TIMEOUT = 20
 ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'master'
+ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
 
 
 LOGGER = configure_logger(__name__)
@@ -73,6 +74,10 @@ def __init__(self, **kwargs):
             "default_scm_branch",
             ADF_DEFAULT_SCM_FALLBACK_BRANCH,
         )
+        self.default_scm_codecommit_account_id = self.map_params.get(
+            "/adf/scm/default_scm_codecommit_account_id",
+            ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID,
+        )
         self.configuration = self._generate_configuration()
         self.config = self.generate()
 
@@ -730,6 +735,10 @@ def __init__(
             "default_scm_branch",
             ADF_DEFAULT_SCM_FALLBACK_BRANCH,
         )
+        self.default_scm_codecommit_account_id = map_params.get(
+            "/adf/scm/default_scm_codecommit_account_id",
+            ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID,
+        )
         self.cfn = _codepipeline.CfnPipeline(
             self,
             'pipeline',

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py

@@ -82,7 +82,7 @@
 }
 GITHUB_SOURCE = {
     "provider": 'github',
-    "properties": GITHUB_SOURCE_PROPS
+    Optional("properties"): CODECOMMIT_SOURCE_PROPS
 }
 
 # CodeStar Source
@@ -282,13 +282,23 @@
     'codebuild': Schema(DEFAULT_CODEBUILD_BUILD),
 }
 PROVIDER_SCHEMA = {
-    'source': And(
-        {
-            'provider': Or('codecommit', 'github', 's3', 'codestar'),
-            'properties': dict,
-        },
-        # pylint: disable=W0108
-        lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
+    'source': Or(
+        And(
+            {
+                'provider': Or('github', 's3', 'codestar'),
+                'properties': dict,
+            },
+            # pylint: disable=W0108
+            lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
+        ),
+        And(
+            {
+                'provider': Or('codecommit'),
+                Optional('properties'): dict,
+            },
+            # pylint: disable=W0108
+            lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
+        ),
     ),
     Optional('build'): And(
         {
@@ -333,7 +343,10 @@
 TARGET_SCHEMA = {
     Optional("path"): Or(str, int, TARGET_LIST_SCHEMA),
     Optional("tags"): {
-        And(str, Regex(r"\A.{1,128}\Z")): And(str, Regex(r"\A.{0,256}\Z"))
+        And(str, Regex(r"\A.{1,128}\Z")): Or(
+            And(str, Regex(r"\A.{0,256}\Z")),
+            And(list)
+        )
     },
     Optional("target"): Or(str, int, TARGET_LIST_SCHEMA),
     Optional("name"): str,