diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 3b43ad23..792c5205 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -107,7 +107,7 @@ "xattr", "smartbrain" ], - "ignoreWords": ["ACCOUNTNAME","OIDCISSUERURL","UNIQUESTRING", "outfile"], + "ignoreWords": ["ACCOUNTNAME","OIDCISSUERURL","UNIQUESTRING", "outfile","contoso"], "import": [], "enableFiletypes": [ "!css", diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-EID/ad_groups.ps1 b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-EID/ad_groups.ps1 new file mode 100644 index 00000000..070dc1ab --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-EID/ad_groups.ps1 @@ -0,0 +1,20 @@ +param( + [Parameter(Mandatory=$true)] + [string]$appdevs, + [Parameter(Mandatory=$true)] + [string]$aksops +) + +# checking if Azure module is installed +$isInstalled = $false +if(Get-InstalledModule -Name Az.Resources -ErrorAction SilentlyContinue){ + $isInstalled = $true +} + +if($isInstalled){ + New-AzADGroup -DisplayName $appdevs -MailNickname $appdevs + New-AzADGroup -DisplayName $aksops -MailNickname $aksops +} +else { + Write-Output "Azure PowerShell not installed. Installation steps in: https://learn.microsoft.com/powershell/azure/install-az-ps" +} diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-eid.md b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-eid.md new file mode 100644 index 00000000..2435aa63 --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-eid.md @@ -0,0 +1,81 @@ +# Prerequisites and Microsoft Entra ID + +This is the starting point for the instructions on deploying the [AKS Baseline private cluster reference implementation](../README.md). There is required access and tooling you'll need in order to accomplish this. Follow the instructions below and on the subsequent pages so that you can get your environment ready to proceed with the AKS cluster creation. + +## Steps + +1. Latest [Azure CLI installed](https://learn.microsoft.com/cli/azure/install-azure-cli?view=azure-cli-latest) (must be at least 2.59), or you can perform this from Azure Cloud Shell by clicking below. +1. An Azure subscription. + + The subscription used in this deployment cannot be a [free account](https://azure.microsoft.com/free); it must be a standard EA, pay-as-you-go, or Visual Studio benefit subscription. This is because the resources deployed here are beyond the quotas of free subscriptions. + + > :warning: The user or service principal initiating the deployment process _must_ have the following minimal set of Azure Role-Based Access Control (RBAC) roles: + > + > * [Contributor role](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#contributor) is _required_ at the subscription level to have the ability to create resource groups and perform deployments. + > * [User Access Administrator role](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator) is _required_ at the subscription level since you'll be performing role assignments to managed identities across various resource groups. + +1. **This step only applies if you are creating a new Microsoft Entra group for this deployment. If you have one already existing and you are a part of it, you can skip this prerequisite, and the remaining steps in this page, move on to the next page by clicking on the link at the bottom**. + + A Microsoft Entra ID tenant to associate your Kubernetes RBAC Cluster API authentication to. + + > :warning: The user or service principal initiating the deployment process _must_ have the following minimal set of Microsoft Entra ID permissions assigned: + > + > * Microsoft Entra [User Administrator](https://learn.microsoft.com/entra/identity/role-based-access-control/permissions-reference#user-administrator-permissions) is _required_ to create a "break glass" AKS admin Microsoft Entra security group and user. Alternatively, you could get your Microsoft Entra admin to create this for you when instructed to do so. + > * If you are not part of the User Administrator group in the tenant associated to your Azure subscription, please consider [creating a new tenant](https://learn.microsoft.com/entra/fundamentals/create-new-tenant#create-a-new-tenant-for-your-organization) to use while evaluating this implementation. The Microsoft Entra tenant backing your cluster's API RBAC does NOT need to be the same tenant associated with your Azure subscription. + +# Create Microsoft Entra groups for AKS + +Before creating the Microsoft Entra ID integrated cluster, groups must be created that can be later mapped to the Built-In Roles of "Azure Kubernetes Service Cluster User Role" and "Azure Kubernetes Service RBAC Cluster Admin". + +Depending on the needs of your organization, you may have a choice of existing groups to use or a new groups may need to be created for each cluster deployment. + +Navigate to "/AKS-Secure-Baseline-Private-AVM/Bicep/02-EID" folder + +```azurecli +cd ./Scenarios/AKS-Secure-Baseline-Private-AVM/Bicep/02-EID +``` + +Use the Azure CLI or Azure PowerShell to create the Microsoft Entra groups. Replace the Microsoft Entra group names below with the name of the Microsoft Entra groups you want to create, such as AKS_ES_dev, AKS_ES_ops. There should be no space in the names. + +# [CLI](#tab/CLI) + +```azurecli +appdevs= +aksops= + +az ad group create --display-name $appdevs --mail-nickname $appdevs +az ad group create --display-name $aksops --mail-nickname $aksops +``` + +# [PowerShell](#tab/PowerShell) + +Running the command to create the new Microsoft Entra groups requires the New-AzADGroup cmdlet. More details can be found [here](https://learn.microsoft.com/powershell/azure/install-az-ps). + +Install New-AzADGroup cmdlet + +```azurepowershell +Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser +Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -Force +``` + +Run the command below to create two new Microsoft Entra groups in your tenant. + +```azurepowershell +./ad_groups.ps1 -appdevs -aksops +``` + +## Ensure you are part of the Microsoft Entra group you just created or pointed to + +1. Go to Azure portal and type Microsoft Entra ID +2. Select **Microsoft Entra ID** +3. Click on **Groups** in the left blade +4. Select the Admin User group you just created. For the default name, this should be *AKS App Admin Team* +5. Click on **Members** in the left blade +6. ![Location of private link for keyvault](../media/adding-to-eid-group.png) +7. Click **+ Add members** +8. Enter your name in the search bar and select your user(s) +9. Click **Select** + +### Next step + +:arrow_forward: [Creation of Hub Network & its respective Components](./03-network-hub.md) diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/03-network-hub.md b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/03-network-hub.md new file mode 100644 index 00000000..9a110325 --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/03-network-hub.md @@ -0,0 +1,41 @@ +# Deploy this scenario using the AKS AVM + +This scenario will be deployed using Azure Verified Modules (AVM). AVM is an initiative to consolidate and set the standards for what a good Infrastructure-as-Code module looks like. + +Modules will then align to these standards, across languages (Bicep, Terraform etc.) and will then be classified as AVMs and available from their respective language specific registries. These AVMs are fully supported by Microsoft and customers can use them in their production Terraform Code. For more information about AVM, check out the [AVM website](https://azure.github.io/Azure-Verified-Modules/). + +# Create the Hub Network + +If you haven't yet, clone the repo and cd to the appropriate folder + +```bash +git clone https://github.com/Azure/AKS-Landing-Zone-Accelerator +cd ./Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/02-EID +``` + +The following will be created: + +* Resource Group for Hub Networking +* Hub VNET +* Azure Firewall +* Azure Bastion Host + +Navigate to "/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/" folder + +```bash +cd ./03-Network-Hub +``` + +Review the "input.tf" file and update the variable values if required according to your needs. Pay attentions to VNET address prefixes and subnets so it doesn't overlap Spoke VNET in further steps. Also, please pay attention to update Subnet prefix for AKS cluster in Spoke VNET in the further steps to be planned and update in this file. + +Once the files are updated, deploy using terraform cli. + +# [CLI](#tab/CLI) + +```terracli +terraform init +terraform plan -out main.tfplan +terraform apply main.tfplan -auto-approve +``` + +:arrow_forward: [Creation of Spoke Network & its respective Components](./04-network-lz.md) diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/04-network-lz.md b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/04-network-lz.md new file mode 100644 index 00000000..33775a8c --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/04-network-lz.md @@ -0,0 +1,28 @@ +# Create the Landing Zone Network + +The following will be created: + +* Resource Group for Landing Zone Networking +* Spoke Virtual Network and Subnets +* Peering of Hub and Spoke Networks +* Private DNS Zones +* Application Gateway +* NSGs for AKS subnet and Application Gateway subnet + +Navigate to "/Scenarios/AKS-Secure-Baseline-PrivateCluster-AVM/Terraform/" folder + +```bash +cd ./04-Network-LZ +``` + +Review "input.tf" and update the variable values as required. Please note to verify the Azure Firewall Private IP from the previous deployment in step 03. Once the files are updated, deploy using terraform cli. + +# [CLI](#tab/CLI) + +```terracli +terraform init +terraform plan -out main.tfplan +terraform apply main.tfplan -auto-approve +``` + +:arrow_forward: [Creation of Supporting Components for AKS](./05-aks-supporting.md) diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/05-aks-supporting.md b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/05-aks-supporting.md new file mode 100644 index 00000000..88c6b6ac --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/05-aks-supporting.md @@ -0,0 +1,27 @@ +# Create resources that support AKS + +The following will be created: + +* Azure Container Registry +* Azure Key Vault +* Private Link Endpoints for ACR and Key Vault +* Related DNS settings for private endpoints +* A managed identity + +Navigate to "/Scenarios/AKS-Secure-Baseline-PrivateCluster/Terraform/" folder + +```bash +cd ./05-AKS-supporting +``` + +Review "input.tf" and update the variable values as required. Once the files are updated, deploy using terraform cli. + +# [CLI](#tab/CLI) + +```terracli +terraform init +terraform plan -out main.tfplan +terraform apply main.tfplan -auto-approve +``` + +:arrow_forward: [Creation of AKS & enabling Addons](./06-aks-cluster.md) diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/input.tf b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/input.tf index b63d401a..56bc0d6b 100644 --- a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/input.tf +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/input.tf @@ -24,9 +24,9 @@ variable "vnetHubName" { default = "vnet-hub" } -variable "admin-group-object-ids" { +variable "adminGroupObjectIds" { type = string - default = "d1553d93-3b9f-4d52-a28b-e4a4a27c114c" + default = " " } diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/main.tf b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/main.tf index ca61d537..3de1213d 100644 --- a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/main.tf +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-AKS-Cluster/main.tf @@ -105,7 +105,7 @@ resource "azurerm_kubernetes_cluster" "aks-cluster" { } azure_active_directory_role_based_access_control { managed = true - admin_group_object_ids = [var.admin-group-object-ids] + admin_group_object_ids = [var.adminGroupObjectIds] } default_node_pool { diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-aks-cluster.md b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-aks-cluster.md new file mode 100644 index 00000000..e45b33e5 --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/06-aks-cluster.md @@ -0,0 +1,82 @@ +# Create resources for the AKS Cluster + +The following will be created: + +* AKS Cluster with KeyVault, nginx and monitoring addons +* Log Analytics Workspace +* ACR Access to the AKS Cluster +* Updates to KeyVault access policy with AKS keyvault addon + +Navigate to "/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/" folder + +```bash +cd ./06-AKS-cluster +``` + +To create an AKS cluster that can use the Secrets Store CSI Driver, you must enable the AKS-AzureKeyVaultSecretsProvider feature flag on your subscription. Register the AKS-AzureKeyVaultSecretsProvider feature flag by using the az feature register command, as shown below + +```bash +az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService/AKS-AzureKeyVaultSecretsProvider')].{Name:name,State:properties.state}" +``` + +if not enter the command below to enable it + +```bash +az feature register --namespace "Microsoft.ContainerService" --name "AKS-AzureKeyVaultSecretsProvider" +``` + +It takes a few minutes for the status to show *Registered*. Verify the registration status by using the [az feature list](https://learn.microsoft.com/cli/azure/feature#az_feature_list) command: + +```bash +az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService/AKS-AzureKeyVaultSecretsProvider')].{Name:name,State:properties.state}" +``` + +When ready, refresh the registration of the *Microsoft.ContainerService* resource provider by using the [az provider register](https://learn.microsoft.com/cli/azure/provider#az_provider_register) command: + +```bash +az provider register --namespace Microsoft.ContainerService +``` + +There are a few additional Azure Providers and features that needs to be registered as well. Follow the same steps above for the following providers and features: + +* Microsoft.ContainerService +* AKS-AzureKeyVaultSecretsProvider +* Microsoft.OperationsManagement +* Microsoft.OperationalInsights +* EncryptionAtHost + +Here is a list with all required providers or features to be registered: + +```bash +az provider register --namespace Microsoft.ContainerService +az provider register --namespace Microsoft.OperationsManagement +az provider register --namespace Microsoft.OperationalInsights +az feature register --namespace "Microsoft.ContainerService" --name "AKS-AzureKeyVaultSecretsProvider" +az feature register --namespace Microsoft.Compute --name EncryptionAtHost +``` + +> :warning: Don't move ahead to the next steps until all providers are registered. + +There is a admin group you need to change in input.tf + + - Admin group which will grant the role "Azure Kubernetes Service Cluster Admin Role". The variable name is: admin-group-object-ids. + +## Deploy the cluster + +Review "**input.tf**" file and update the variable values as required. Please make sure to update the Microsoft Entra ID group IDs with ones created in Step 02 and kubernetesVersion in the variables file. Once the files are updated, deploy using terraform cli + +The Kubernetes community releases minor versions roughly every three months. AKS has it own supportability policy based in the community releases. Before proceeding with the deployment, check the latest version reviewing the [supportability doc](https://learn.microsoft.com/azure/aks/supported-kubernetes-versions). You can also check the latest version by using the following command: + +```azurecli +az aks get-versions -l $REGION +``` + +# [CLI](#tab/CLI) + +```terracli +terraform init +terraform plan -out main.tfplan +terraform apply main.tfplan -auto-approve +``` + +:arrow_forward: [Deploy a Basic Workload](./07-workload.md) diff --git a/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/07-worload.md b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/07-worload.md new file mode 100644 index 00000000..55b28247 --- /dev/null +++ b/Scenarios/AKS-Secure-Baseline-Private-AVM/Terraform/07-worload.md @@ -0,0 +1,293 @@ +# Deploy a Basic Workload using the AKS-Store-Demo Application + +This application consists of a group of containerized microservices that can be easily deployed into an Azure Kubernetes Service (AKS) cluster. This is meant to show a realistic scenario using a polyglot architecture, event-driven design, and common open source back-end services (eg - RabbitMQ, MongoDB). The application also leverages OpenAI's GPT-3 models to generate product descriptions. You can find out more about the application at . + +As the infrastructure has been deployed in a private AKS cluster setup with private endpoints for the container registry and other components, you will need to perform the application container build and the publishing to the Container Registry from the Dev Jumpbox in the Hub VNET, connecting via the Bastion Host service. + + If your computer is connected to the hub network, you may be able to just use that as well. The rest of the steps can be performed on your local machine by using AKS Run commands which allow access into private clusters using RBAC. This will help with improving security and will provide a more user-friendly way of editing YAML files. + +## Connecting to the Bastion Host + +The first major step to deploying the application is to connect to the jumpbox inside the private network and authenticate to Azure and the AKS cluster. + +1. From the *jumpbox* resource in the *AksTerra-AVM-LZ-RG* resource group, connect to the VM using the **Connect via Bastion**. + +1. If prompted, allow the browser to read the contents of your clipboard. + +1. From the jumpbox command line, clone the *aks-landing-Zone-Accelerator* repository which contains some setup scripts needed shortly. + + ```bash + git clone https://github.com/Azure/AKS-Landing-Zone-Accelerator/ + ``` + +1. Run the setup script to apply the latest updates to the jumpbox and to install other required packages. + + ```bash + cd AKS-Landing-Zone-Accelerator/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/07-Workload + + chmod +x script.sh + + sudo ./script.sh + ``` + + NOTE: You might need to hit Enter when it says "Restarting services..." + +1. Login to Azure and select your subscription + + ```bash + TENANTID= + + az login -t $TENANTID + ``` + + If your account has access to multiple subscriptions, you will be prompted to select the one you wish to use. + +1. If you selected the wrong subscription, it can be set correctly as shown. + + ```bash + az account set --subscription + ``` + +1. Set environment variables + + ```bash + # Enter the name of your ACR below + SPOKERG=AKS-LZA-SPOKE + AKSCLUSTERNAME=$(az aks list -g $SPOKERG --query [0].name -o tsv) + ACRNAME=$(az acr list -g $SPOKERG --query [0].name -o tsv) + ``` + + Now login a second time whilst sudo'ed as root. *This is to get around a problem later where an Azure Container Registry command needs access to AZ access tokens AND the Docker Daemon at the same time - it makes installation easier if that one command runs as root.* + +1. To control Kubernetes directly from the jumpbox, *kubectl* and the *kubelogin* commands must be installed. + + ```bash + sudo snap install kubectl --classic + + sudo az aks install-cli + ``` + +1. Download from Azure the configuration file for connecting to AKS. + + ```bash + az aks get-credentials --name $AKSCLUSTERNAME --resource-group $SPOKERG + ``` + +1. Test the connection by requesting a list of nodes in the cluster (you will be asked to login again so that you can obtain an AKS specific token). + + ```bash + kubectl get nodes + ``` + +### Control the default NGINX ingress controller configuration (preview) + +As part of deploying our AKS environment, we enabled the [AKS app routing addon](https://learn.microsoft.com/en-us/azure/aks/app-routing). For better security, we will ensure that our applications, including the ingress controller are only available within the internal network of your organization. We will later expose our application to the internet using a web application firewall enabled application gateway. Our first step is to ensure that our default settings for the nginx ingress controller managed by the AKS app routing addon ensures the ingress has only internal IP addresses. As of the time of writing, this is a preview feature that requires the use of aks-preview Azure CLI extension. If you do not have this installed, use the commands below to install it. + +```bash +az extension add --name aks-preview +``` + +If you have a version of AKS-preview that is version 7.0.0b5 or later, you can just update it. + +```bash +az extension update --name aks-preview +``` + +In addition, some preview features require you to register them. For example, to register the Deployment Safeguards feature + +```bash +az feature register --namespace Microsoft.ContainerService --name SafeguardsPreview +``` + +Once this feature is registered, refresh the registration of the Microsoft.ContainerService provider + +```bash +az provider register --namespace Microsoft.ContainerService +``` + +Now that you have enabled the preview feature, run the command below to update the default configuration of your app routing addon so that by default, it deploys ingress controllers with internal ip addresses. + +```bash +az aks approuting update --resource-group $SPOKERG --name $AKSCLUSTERNAME --nginx Internal +``` + +## Build Container Images + +Clone the sample application Git Repo to the Dev Jumpbox: + +1. The AKS Store Demo repo: + +```bash +git clone https://github.com/Azure-Samples/aks-store-demo +``` + +Navigate to each application code directory, build and tag the containers with the name of your Azure Container Registry and push the images to ACR. + +*NOTE: If you are deploying to Azure US Government, use '.azurecr.us' instead of '.azurecr.io' in the commands below.* + +```bash + +cd aks-store-demo/src + +# Change directory into each app folder and build/tag the image. Example: +cd ai-service +sudo docker build . -t $ACRNAME.azurecr.io/ai-service:v1 + +# Do this for each app in the directory, there should be 8 in total. Remember to change the tag name for each folder: + +# e.g. +# cd makeline-service +# sudo docker build . -t $ACRNAME.azurecr.io/makeline-service:v1 +``` + +Now check all container images have built correctly: + +```bash +sudo docker images +``` + +You should see output similar to + +```bash +REPOSITORY TAG IMAGE ID CREATED SIZE +eslzacrguilfdnvzjuum.azurecr.io/virtual-worker v1 0d6da98b7a1f 12 minutes ago 97MB +eslzacrguilfdnvzjuum.azurecr.io/virtual-customer v1 a07be343f9d4 13 minutes ago 96.7MB +eslzacrguilfdnvzjuum.azurecr.io/store-front v1 692284db83ac 15 minutes ago 16.6MB +eslzacrguilfdnvzjuum.azurecr.io/store-admin v1 9fd83b91a176 17 minutes ago 15MB +eslzacrguilfdnvzjuum.azurecr.io/product-service v1 2056e083ede1 18 minutes ago 121MB +eslzacrguilfdnvzjuum.azurecr.io/order-service v1 6d68a60bacc4 25 minutes ago 172MB +eslzacrguilfdnvzjuum.azurecr.io/makeline-service v1 1a0232d81f29 26 minutes ago 27.6MB +eslzacrguilfdnvzjuum.azurecr.io/ai-service v1 fddf58277b93 29 minutes ago 431MB +``` + +## Log into Azure Container Registry + +You must now login to the ACR to upload the new images. + +> Notice this is being run as root because the command needs access to the Docker daemon (this is why you had to login twice earlier - once as 'azureuser' and once as 'root'). + +```bash +sudo az acr login -n $ACRNAME +# Login Succeeded +``` + +> NOTE: If this fails and requires providing username and password, you might have to log into your Azure Portal and head to the ACR instance. On the left panel under settings, click on Access Keys. You will see the admin username and password there if Admin user is enabled. + +## Push the images to the container registry + +```bash +for i in $(sudo docker images | awk 'NR>1 { print $1}') ; do + (echo "Pushing $i" && sudo docker push $i:v1) +done +``` + +As well as the custom images uploaded above, there are additional images which we can just import from a public repository. Import these using the `az acr import` command: + +```bash +az acr import --name $ACRNAME --source mcr.microsoft.com/mirror/docker/library/mongo:4.2 --image mongo:4.2 + +az acr import --name $ACRNAME --source mcr.microsoft.com/mirror/docker/library/rabbitmq:3.10-management-alpine --image rabbitmq:3.10-management-alpine +``` + +Ensure your ACR now has all the images you need by running the command below + +```bash +az acr repository list --name $ACRNAME --output table + +# Result +# ---------------- +# ai-service +# makeline-service +# mongo +# order-service +# product-service +# rabbitmq +# store-admin +# store-front +# virtual-customer +# virtual-worker +``` + +You should also connect your AKS Cluster to the Azure Container Registry (ACR) so when it attempts to pull images it can authenticate correctly: + +```bash +az aks update --name $AKSCLUSTERNAME --resource-group $SPOKERG --attach-acr $ACRNAME +``` + +Now deploy the application using the HELM chart. Make sure to update the value of the containerRegistry in the command below to your ACR name: + +```bash +cd $HOME/AKS-Landing-Zone-Accelerator/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/07-Workload + +helm install monkey-magic ./shoppingDemo --set containerRegistry=$ACRNAME.azurecr.io +# apply the ingress controller +kubectl apply -f shoppingDemo/templates/ingress-via-nginx-internal.yaml +``` + +After deployment, check the pods have created correctly: + +```bash +kubectl get pods +``` + +A correct installation looks like this: + +```bash +NAME READY STATUS RESTARTS AGE +makeline-service-57c7b44d6b-mqc97 1/1 Running 0 107s +mongodb-0 1/1 Running 0 107s +order-service-6df845965-8kg27 1/1 Running 0 107s +product-service-79f7cc5cd-fw6r2 1/1 Running 0 107s +rabbitmq-0 1/1 Running 0 107s +store-admin-6d5cf5676-9cmrj 1/1 Running 0 107s +store-front-56b745cbf-57f27 1/1 Running 0 107s +virtual-customer-59d74777d6-qvwkd 1/1 Running 0 107s +virtual-worker-69576c848b-49g24 1/1 Running 0 107s +``` + +```bash +kubectl get ingress +``` + +```bash +NAME CLASS HOSTS ADDRESS PORTS AGE +internal-ingress webapprouting.kubernetes.azure.com private.contoso.com 10.1.1.10 80 14s +``` + +### Testing the application internally + +Your ingress controller is accessible from within the virtual network but not from the internet. Private DNS Zone has been configured with hostname `private.contoso.com`. + +Use `curl` command to test that the application is running in the cluster and the ingress was configured properly + +```bash +curl http://private.contoso.com +``` + +```bash +store-front