Bug fixing and refactoring for e2e tests (#107)

# Pull Request

## Description

This PR is part of the major refactoring work to de-couple the bootstrap
and support bicep automation.

The fixes and refactoring in this branch were designed to make the e2e
tests pass successfully.

You can see the successful test run for this branch here: 

- PR: Azure/accelerator-bootstrap-modules#1
- Test Run:
https://github.com/Azure/accelerator-bootstrap-modules/actions/runs/8288999772/job/22684596007?pr=1

## License

By submitting this pull request, I confirm that my contribution is made
under the terms of the projects associated license.

actions_bootstrap_for_e2e_tests.ps1

@@ -0,0 +1,37 @@
+# Bootstrap dependencies
+
+# https://docs.microsoft.com/powershell/module/packagemanagement/get-packageprovider
+Get-PackageProvider -Name Nuget -ForceBootstrap | Out-Null
+
+# https://docs.microsoft.com/powershell/module/powershellget/set-psrepository
+Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
+
+# List of PowerShell Modules required for the build
+$modulesToInstall = [System.Collections.ArrayList]::new()
+# https://github.com/nightroman/Invoke-Build
+$null = $modulesToInstall.Add(([PSCustomObject]@{
+            ModuleName    = 'InvokeBuild'
+            ModuleVersion = '5.10.2'
+        }))
+
+'Installing PowerShell Modules'
+foreach ($module in $modulesToInstall) {
+    $installSplat = @{
+        Name               = $module.ModuleName
+        RequiredVersion    = $module.ModuleVersion
+        Repository         = 'PSGallery'
+        SkipPublisherCheck = $true
+        Force              = $true
+        ErrorAction        = 'Stop'
+    }
+    try {
+        Install-Module @installSplat
+        Import-Module -Name $module.ModuleName -ErrorAction Stop
+        '  - Successfully installed {0}' -f $module.ModuleName
+    } catch {
+        $message = 'Failed to install {0}' -f $module.ModuleName
+        "  - $message"
+        throw
+    }
+}
+

src/ALZ.build.ps1

@@ -64,6 +64,9 @@ Add-BuildTask HelpLocal Clean, ImportModuleManifest, CreateHelpStart
 #Full build sans integration tests
 Add-BuildTask BuildNoIntegration -Jobs $str2
 
+#Build and Install Only
+Add-BuildTask BuildAndInstallOnly Clean, ImportModuleManifest, Build, Archive, Install
+
 # Pre-build variables to be used by other portions of the script
 Enter-Build {
     $script:ModuleName = (Split-Path -Path $BuildFile -Leaf).Split('.')[0]

src/ALZ/Private/Convert-HCLVariablesToUserInputConfig.ps1 → src/ALZ/Private/Config-Helpers/Convert-HCLVariablesToUserInputConfig.ps1

@@ -55,6 +55,7 @@ function Convert-HCLVariablesToUserInputConfig {
             $inputType = "UserInput"
             if($allComputedInputs) {
                 $inputType = "ComputedInput"
+                Write-Verbose "Name: $($variable.Name), Has Validation: $hasValidation, Order: $order, ValidationType: $validationType, Description: $description, InputType: $inputType"
             }
 
             $sensitive = $false
@@ -90,6 +91,7 @@ function Convert-HCLVariablesToUserInputConfig {
             }
 
             if($hasValidation) {
+                Write-Verbose "Validation: $hasValidation - $validationType"
                 $validator = $validators.PSObject.Properties[$validationType].Value
                 $description = "$description ($($validator.Description))"
                 if($validator.Type -eq "AllowedValues"){

src/ALZ/Private/Get-ALZConfig.ps1 → src/ALZ/Private/Config-Helpers/Get-ALZConfig.ps1

@@ -39,6 +39,6 @@ function Get-ALZConfig {
         throw "The config file must be a json or yaml/yml file"
     }
 
-    Write-Verbose "Config file loaded from $configFilePath with $($config.PSObject.Properties.Count) properties."
+    Write-Verbose "Config file loaded from $configFilePath with $($config.PSObject.Properties.Name.Count) properties."
     return $config
 }

src/ALZ/Private/Request-SpecialInput.ps1 → src/ALZ/Private/Config-Helpers/Request-SpecialInput.ps1

@@ -71,10 +71,18 @@ function Request-SpecialInput {
                 Write-InformationColored "Please select the bootstrap module you would like to use, you can enter one of the following keys:" -ForegroundColor Yellow -InformationAction Continue
 
                 $bootstrapOptions = @()
-                foreach ($bootstrapModule in $bootstrapModules.PsObject.Properties) {
-                    Write-InformationColored "- $($bootstrapModule.Name) ($($bootstrapModule.Value.description))" -ForegroundColor Yellow -InformationAction Continue
-                    $bootstrapOptions += $bootstrapModule.Name
+                if($bootstrapModules.PsObject.Properties.Name.Count -eq 0) {
+                    $bootstrapOptions += "azuredevops"
+                    Write-InformationColored "- azuredevops" -ForegroundColor Yellow -InformationAction Continue
+                    $bootstrapOptions += "github"
+                    Write-InformationColored "- github" -ForegroundColor Yellow -InformationAction Continue
+                } else {
+                    foreach ($bootstrapModule in $bootstrapModules.PsObject.Properties) {
+                        Write-InformationColored "- $($bootstrapModule.Name) ($($bootstrapModule.Value.description))" -ForegroundColor Yellow -InformationAction Continue
+                        $bootstrapOptions += $bootstrapModule.Name
+                    }
                 }
+
                 Write-InformationColored ": " -ForegroundColor Yellow -NoNewline -InformationAction Continue
                 $result = Read-Host
 

src/ALZ/Private/Deploy-Accelerator-Helpers/Get-BootstrapAndStarterConfig.ps1

@@ -0,0 +1,84 @@
+
+function Get-BootstrapAndStarterConfig {
+    [CmdletBinding(SupportsShouldProcess = $true)]
+    param(
+        [Parameter(Mandatory = $false)]
+        [string]$iac,
+        [Parameter(Mandatory = $false)]
+        [string]$bootstrap,
+        [Parameter(Mandatory = $false)]
+        [string]$bootstrapPath,
+        [Parameter(Mandatory = $false)]
+        [string]$bootstrapConfigPath,
+        [Parameter(Mandatory = $false)]
+        [PSCustomObject]$userInputOverrides
+    )
+
+    if ($PSCmdlet.ShouldProcess("Get Configuration for Bootstrap and Starter", "modify")) {
+        $hasStarterModule = $false
+        $starterModuleUrl = ""
+        $starterModuleSourceFolder = ""
+        $starterReleaseTag = ""
+        $starterPipelineFolder = ""
+
+        $bootstrapDetails = $null
+        $validationConfig = $null
+        $inputConfig = $null
+
+        # Get the bootstap configuration
+        $bootstrapConfigFullPath = Join-Path $bootstrapPath $bootstrapConfigPath
+        Write-Verbose "Bootstrap config path $bootstrapConfigFullPath"
+        $bootstrapConfig = Get-ALZConfig -configFilePath $bootstrapConfigFullPath
+        $validationConfig = $bootstrapConfig.validators
+
+        # Get the available bootstrap modules
+        $bootstrapModules = $bootstrapConfig.bootstrap_modules
+
+        # Request the bootstrap type if not already specified
+        if($bootstrap -eq "") {
+            $bootstrap = Request-SpecialInput -type "bootstrap" -bootstrapModules $bootstrapModules -userInputOverrides $userInputOverrides
+        }
+
+        # Get the bootstrap details and validate it exists (use alias for legacy values)
+        $bootstrapDetails = $bootstrapModules.PsObject.Properties | Where-Object { $_.Name -eq $bootstrap -or $bootstrap -in $_.Value.aliases }
+        if($null -eq $bootstrapDetails) {
+            Write-InformationColored "The bootstrap type '$bootstrap' that you have selected does not exist. Please try again with a valid bootstrap type..." -ForegroundColor Red -InformationAction Continue
+            throw
+        }
+
+        # Get the starter modules for the selected bootstrap if it has any
+        $bootstrapStarterModule = $bootstrapDetails.Value.PSObject.Properties | Where-Object { $_.Name -eq  "starter_modules" }
+
+        if($null -ne $bootstrapStarterModule) {
+            # If the bootstrap has starter modules, get the details and url
+            $hasStarterModule = $true
+            $starterModules = $bootstrapConfig.PSObject.Properties | Where-Object { $_.Name -eq "starter_modules" }
+            $starterModuleType = $bootstrapStarterModule.Value
+            $starterModuleDetails = $starterModules.Value.PSObject.Properties | Where-Object { $_.Name -eq $starterModuleType }
+            if($null -eq $starterModuleDetails) {
+                Write-InformationColored "The starter modules '$($starterModuleType)' for the bootstrap type '$bootstrap' that you have selected does not exist. This could be an issue with your custom configuration, please check and try again..." -ForegroundColor Red -InformationAction Continue
+                throw
+            }
+
+            $starterModuleUrl = $starterModuleDetails.Value.$iac.url
+            $starterModuleSourceFolder = $starterModuleDetails.Value.$iac.module_path
+            $starterPipelineFolder = $starterModuleDetails.Value.$iac.pipeline_folder
+        }
+
+        # Get the bootstrap interface user input config
+        $inputConfigFilePath = Join-Path -Path $bootstrapPath -ChildPath $bootstrapDetails.Value.interface_config_file
+        Write-Verbose "Interface config path $inputConfigFilePath"
+        $inputConfig = Get-ALZConfig -configFilePath $inputConfigFilePath
+
+        return @{
+            bootstrapDetails          = $bootstrapDetails
+            hasStarterModule          = $hasStarterModule
+            starterModuleUrl          = $starterModuleUrl
+            starterModuleSourceFolder = $starterModuleSourceFolder
+            starterReleaseTag         = $starterReleaseTag
+            starterPipelineFolder     = $starterPipelineFolder
+            validationConfig          = $validationConfig
+            inputConfig               = $inputConfig
+        }
+    }
+}

src/ALZ/Private/New-Bootstrap.ps1 → src/ALZ/Private/Deploy-Accelerator-Helpers/New-Bootstrap.ps1

@@ -43,6 +43,9 @@ function New-Bootstrap {
 
     if ($PSCmdlet.ShouldProcess("ALZ-Terraform module configuration", "modify")) {
 
+        $bootstrapPath = Join-Path $bootstrapTargetPath $bootstrapRelease
+        $starterPath = Join-Path $starterTargetPath $starterRelease
+
         # Setup tools
         $hclParserToolPath = Get-HCLParserTool -alzEnvironmentDestination $bootstrapPath -toolVersion "v0.6.0"
 
@@ -57,8 +60,6 @@ function New-Bootstrap {
         $starterCachePath = Join-Path -Path $starterPath -ChildPath $starterCacheFileName
         $starterCachedConfig = Get-ALZConfig -configFilePath $starterCachePath
 
-        $bootstrapPath = Join-Path $bootstrapTargetPath $bootstrapRelease
-        $starterPath = Join-Path $starterTargetPath $starterRelease
         $bootstrapModulePath = Join-Path -Path $bootstrapPath -ChildPath $bootstrapDetails.Value.location
 
         Write-Verbose "Bootstrap Module Path: $bootstrapModulePath"
@@ -82,8 +83,11 @@ function New-Bootstrap {
 
         if($hasStarter) {
             $starter = Request-SpecialInput -type "starter" -starterPath $starterPath -userInputOverrides $userInputOverrides
-            $starterModulePath = Join-Path -Path $starterPath -ChildPath $starter
-            $pipelineModulePath = Join-Path -Path $starterPath -ChildPath $starterPipelineFolder
+            $starterModulePath = Resolve-Path (Join-Path -Path $starterPath -ChildPath $starter)
+            $pipelineModulePath = Resolve-Path (Join-Path -Path $starterPath -ChildPath $starterPipelineFolder)
+
+            Write-Verbose "Starter Module Path: $starterModulePath"
+            Write-Verbose "Pipeline Module Path: $pipelineModulePath"
         }
 
         # Getting the configuration for the interface user input
@@ -109,7 +113,7 @@ function New-Bootstrap {
 
         if($hasStarter) {
             $targetVariableFilePath = Join-Path -Path $starterModulePath -ChildPath "variables.tf"
-            $starterParameters = Convert-HCLVariablesToUserInputConfig -targetVariableFile $targetVariableFilePath -hclParserToolPath $hclParserToolPath -validators $bootstrapConfig.validators
+            $starterParameters = Convert-HCLVariablesToUserInputConfig -targetVariableFile $targetVariableFilePath -hclParserToolPath $hclParserToolPath -validators $validationConfig
         }
 
         # Filter interface inputs if not in bootstrap or starter

src/ALZ/Private/New-FolderStructure.ps1 → src/ALZ/Private/Deploy-Accelerator-Helpers/New-FolderStructure.ps1

@@ -39,7 +39,7 @@ function New-FolderStructure {
             } else {
                 Write-InformationColored "Copying files from $overrideSourceDirectoryPath to $path" -ForegroundColor Green -InformationAction Continue
                 New-Item -Path $path -ItemType "Directory"
-                Copy-Item -Path "$overrideSourceDirectoryPath/$sourceFolder/*" -Destination "$path" -Recurse | Out-String | Write-Verbose
+                Copy-Item -Path "$overrideSourceDirectoryPath/$sourceFolder/*" -Destination "$path" -Recurse -Force | Out-String | Write-Verbose
             }
 
         } else {

src/ALZ/Private/Deploy-Accelerator-Helpers/New-ModuleSetup.ps1

@@ -0,0 +1,37 @@
+
+function New-ModuleSetup {
+    [CmdletBinding(SupportsShouldProcess = $true)]
+    param(
+        [Parameter(Mandatory = $false)]
+        [string]$targetDirectory,
+        [Parameter(Mandatory = $false)]
+        [string]$targetFolder,
+        [Parameter(Mandatory = $false)]
+        [string]$sourceFolder,
+        [Parameter(Mandatory = $false)]
+        [string]$url,
+        [Parameter(Mandatory = $false)]
+        [string]$release,
+        [Parameter(Mandatory = $false)]
+        [string]$moduleOverrideFolderPath,
+        [Parameter(Mandatory = $false)]
+        [bool]$skipInternetChecks
+    )
+
+    if ($PSCmdlet.ShouldProcess("Check and get module", "modify")) {
+        $versionAndPath = $null
+
+        if($skipInternetChecks) {
+            $versionAndPath = Get-ExistingLocalRelease -targetDirectory $targetDirectory -targetFolder $targetFolder
+        } else {
+            $versionAndPath = New-FolderStructure `
+                -targetDirectory $targetDirectory `
+                -url $url `
+                -release $release `
+                -targetFolder $targetFolder `
+                -sourceFolder $sourceFolder `
+                -overrideSourceDirectoryPath $moduleOverrideFolderPath
+        }
+        return $versionAndPath
+    }
+}

src/ALZ/Private/New-ALZEnvironmentBicep.ps1 → src/ALZ/Private/Legacy-Bicep/New-ALZEnvironmentBicep.ps1

@@ -10,12 +10,18 @@ function New-ALZEnvironmentBicep {
         [Parameter(Mandatory = $false)]
         [string] $upstreamReleaseFolderPath,
 
+        [Parameter(Mandatory = $false)]
+        [PSCustomObject] $userInputOverrides = $null,
+
         [Parameter(Mandatory = $false)]
         [ValidateSet("github", "azuredevops")]
         [string] $vcs,
 
         [Parameter(Mandatory = $false)]
-        [switch] $local
+        [switch] $local,
+
+        [Parameter(Mandatory = $false)]
+        [switch] $autoApprove
     )
 
     if ($PSCmdlet.ShouldProcess("ALZ-Bicep module configuration", "modify")) {
@@ -31,14 +37,14 @@ function New-ALZEnvironmentBicep {
         Copy-ALZParametersFile -alzEnvironmentDestination $targetDirectory -upstreamReleaseDirectory $upstreamReleaseFolderPath -configFiles $bicepConfig.config_files | Out-String | Write-Verbose
         Copy-ALZParametersFile -alzEnvironmentDestination $targetDirectory -upstreamReleaseDirectory $upstreamReleaseFolderPath -configFiles $bicepConfig.cicd.$vcs | Out-String | Write-Verbose
 
-        $configuration = Request-ALZEnvironmentConfig -configurationParameters $bicepConfig.parameters
+        $configuration = Request-ALZEnvironmentConfig -configurationParameters $bicepConfig.parameters -userInputOverrides $userInputOverrides -autoApprove:$autoApprove.IsPresent
 
         Set-ComputedConfiguration -configuration $configuration | Out-String | Write-Verbose
         Edit-ALZConfigurationFilesInPlace -alzEnvironmentDestination $targetDirectory -configuration $configuration | Out-String | Write-Verbose
         Build-ALZDeploymentEnvFile -configuration $configuration -Destination $targetDirectory -version $upstreamReleaseVersion | Out-String | Write-Verbose
 
         if($local) {
-            $isGitRepo = Test-ALZGitRepository -alzEnvironmentDestination $targetDirectory
+            $isGitRepo = Test-ALZGitRepository -alzEnvironmentDestination $targetDirectory -autoApprove:$autoApprove.IsPresent
             if (-not $isGitRepo) {
                 Write-InformationColored "The directory $targetDirectory is not a git repository.  Please make sure it is a git repo after initialization." -ForegroundColor Red -InformationAction Continue
             }

src/ALZ/Private/Legacy-Bicep/Test-ALZGitRepository.ps1

@@ -0,0 +1,37 @@
+function Test-ALZGitRepository {
+    [CmdletBinding(SupportsShouldProcess = $true)]
+    param (
+        [Parameter(Mandatory = $true)]
+        [Alias("Output")]
+        [Alias("OutputDirectory")]
+        [Alias("O")]
+        [string] $alzEnvironmentDestination,
+        [Parameter(Mandatory = $false)]
+        [switch] $autoApprove
+    )
+    $gitDirectory = Join-Path $alzEnvironmentDestination ".git"
+    if (Test-Path $gitDirectory) {
+        Write-Verbose "The directory $alzEnvironmentDestination is already a git repository."
+        return $true
+    }
+
+    $runGitInit = $true
+    $gitBranch = "main"
+
+    if(!$autoApprove) {
+        $gitInit = Read-Host "Initialize the directory $alzEnvironmentDestination as a git repository? (y/n)"
+        if ($gitInit -ieq "y") {
+            $runGitInit = $true
+            $gitBranch = Read-Host "Enter the default branch name. (Hit enter to skip and use 'main')"
+            if ($gitBranch -eq "") {
+                $gitBranch = "main"
+            }
+        }
+    }
+
+    if($runGitInit -and $PSCmdlet.ShouldProcess("gitrepository", "initialize")) {
+        git init -b $gitBranch $alzEnvironmentDestination
+    }
+
+    return $runGitInit
+}

src/ALZ/Private/Get-GithubRelease.ps1 → src/ALZ/Private/Shared/Get-GithubRelease.ps1

@@ -124,7 +124,7 @@ function Get-GithubRelease {
 
         Write-Verbose "===> Copying all extracted contents into $targetVersionPath."
 
-        Copy-Item -Path "$($extractedSubFolder.FullName)/$moduleSourceFolder/*" -Destination "$targetVersionPath" -Recurse | Out-String | Write-Verbose
+        Copy-Item -Path "$($extractedSubFolder.FullName)/$moduleSourceFolder/*" -Destination "$targetVersionPath" -Recurse -Force | Out-String | Write-Verbose
 
         Remove-Item -Path "$targetVersionPath/tmp" -Force -Recurse
         Write-InformationColored "The directory for $targetVersionPath has been created and populated." -ForegroundColor Green -InformationAction Continue