Skip to content

Commit

Permalink
feat(workshop): switch ci to oidc tokens (#23)
Browse files Browse the repository at this point in the history
  • Loading branch information
guillaume-chervet authored Dec 13, 2023
1 parent 315e72c commit 66e00cf
Show file tree
Hide file tree
Showing 9 changed files with 184 additions and 67 deletions.
13 changes: 9 additions & 4 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,16 @@ on:
required: true
DOCKER_PASSWORD:
required: true
AZURE_CREDENTIALS:
AZURE_CLIENT_ID:
required: true
AZURE_TENANT_ID:
required: true
AZURE_SUBSCRIPTION_ID:
required: true

jobs:
build:
runs-on: ubuntu-latest

environment: MLOpsPython
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -58,7 +61,9 @@ jobs:
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

- name: download model
run: |
Expand Down
23 changes: 16 additions & 7 deletions .github/workflows/ci.yml → .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@ env:
PNPM_VERSION: 8.5.1
PYTHON_VERSION: 3.10.13
NODE_VERSION: 18
AZURE_RESOURCE_GROUP_NAME: "azure-ml-yola"
AZURE_SUBSCRIPTION_ID: "9d42c9d4-85ab-429d-afb4-4d77f309078c"
AZURE_RESOURCE_GROUP_NAME: "azure-ml-yolo"
AZURE_LOCATION: "northeurope"
AZURE_ML_WORKSPACE_NAME: "cats-dogs-yola"
AZURE_WEBAPP_NAME: "cats-dogs-yola"
DELETE_WEBAPP: "true"
DOCKER_IMAGE_NAME: "guillaumechervet/mlopspython"

permissions:
id-token: write
contents: write
jobs:
lint:
Expand Down Expand Up @@ -146,6 +146,7 @@ jobs:
fi
train:
runs-on: ubuntu-latest
environment: MLOpsPython
needs: tags
outputs:
MODEL_VERSION: ${{ steps.train.outputs.MODEL_VERSION }}
Expand Down Expand Up @@ -173,7 +174,9 @@ jobs:
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Run Train Pipeline
run: |
poetry install
Expand All @@ -191,7 +194,7 @@ jobs:
cd $cwd
#poetry run python azureml_run_test.py > train_output.txt
poetry run python azureml_run_pipeline.py \
--subscription_id ${{ env.AZURE_SUBSCRIPTION_ID }} \
--subscription_id ${{ secrets.AZURE_SUBSCRIPTION_ID }} \
--resource_group_name ${{ env.AZURE_RESOURCE_GROUP_NAME }} \
--workspace_name ${{ env.AZURE_ML_WORKSPACE_NAME }} > train_output.txt
cat train_output.txt
Expand Down Expand Up @@ -265,15 +268,19 @@ jobs:
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
deploy:
runs-on: ubuntu-latest
needs: [tags, build_docker]
steps:
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Deploy container
run: |
# https://learn.microsoft.com/en-us/cli/azure/container?view=azure-cli-latest#az-container-create()
Expand All @@ -296,7 +303,9 @@ jobs:
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Run Integration Test
working-directory: production/integration
run: |
Expand Down
71 changes: 71 additions & 0 deletions bin/init_repository.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# Script based on https://www.techwatching.dev/posts/scripting-azure-ready-github-repository/
# Initialize git repository with current code
# You should have added the main.yml workflow file in the `.github\workflows` directory

#.\init_repository.ps1 -repositoryName "MLOpsPythonMyDemo" -environmentName "MLOpsPython"
param (
[string]$repositoryName = "MLOpsPythonWorkshop",
[string]$environmentName = "MLOpsPython"
)

az login
gh auth login

# Fork MLOPsPython repository
gh repo fork https://github.com/guillaume-chervet/MLOpsPythonDemo1 --default-branch-only --fork-name $repositoryName --clone
cd $repositoryName
git remote remove upstream
git push --set-upstream origin main

# Retrieve the repository full name (org/repo)
$repositoryFullName=$(gh repo view --json nameWithOwner -q ".nameWithOwner")

gh repo set-default "https://github.com/${repositoryFullName}"

# Create environment
gh api --method PUT -H "Accept: application/vnd.github+json" repos/${repositoryFullName}/environments/${environmentName}

# Retrieve the current subscription and current tenant identifiers
$subscriptionId=$(az account show --query "id" -o tsv)
$tenantId=$(az account show --query "tenantId" -o tsv)

# Create an App Registration and its associated service principal
$appId=$(az ad app create --display-name "GitHub Action OIDC for ${repositoryFullName}" --query "appId" -o tsv)
$servicePrincipalId=$(az ad sp create --id $appId --query "id" -o tsv)

# Assign the contributor role to the service principal on the subscription
az role assignment create --role contributor --subscription $subscriptionId --assignee-object-id $servicePrincipalId --assignee-principal-type ServicePrincipal --scope /subscriptions/$subscriptionId

# Prepare parameters for federated credentials
$parametersJson=@{
name = "${environmentName}"
issuer = "https://token.actions.githubusercontent.com"
subject = "repo:${repositoryFullName}:environment:${environmentName}"
description = "Development"
audiences = @(
"api://AzureADTokenExchange"
)
}

# Change parameters to single line string with escaped quotes to make it work with Azure CLI
# https://medium.com/medialesson/use-dynamic-json-strings-with-azure-cli-commands-in-powershell-b191eccc8e9b
$parameters=$($parametersJson | ConvertTo-Json -Depth 100 -Compress).Replace("`"", "\`"")

# Create federated credentials
az ad app federated-credential create --id $appId --parameters $parameters

# Create GitHub secrets needed for the GitHub Actions
gh secret set AZURE_TENANT_ID --body $tenantId --env $environmentName
gh secret set AZURE_SUBSCRIPTION_ID --body $subscriptionId --env $environmentName
gh secret set AZURE_CLIENT_ID --body $appId --env $environmentName
gh secret set DOCKER_PASSWORD --body "robertcarry" --env "$environmentName"
gh secret set DOCKER_USENAME --body "dckr_pat_e2lZ9YgpMt8APE-Qxzn89u6mt28" --env "$environmentName"

# Run workflow
gh workflow enable main.yml
gh workflow run main.yml
$runId=$(gh run list --workflow=main.yml --json databaseId -q ".[0].databaseId")
gh run watch $runId

# Open the repostory in the browser
gh repo view -w
74 changes: 74 additions & 0 deletions bin/init_repository.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Script based on https://www.techwatching.dev/posts/scripting-azure-ready-github-repository/
#.\init_repository.sh "MLOpsPythonMyDemo" "MLOpsPython"

# Définir les paramètres avec des valeurs par défaut
repositoryName="${1:-MLOpsPythonWorkshop}"
environmentName="${2:-MLOpsPython}"

# Authenticate using Azure CLI (az) and GitHub CLI (gh)
az login
gh auth login

# Fork MLOpsPython repository
gh repo fork https://github.com/guillaume-chervet/MLOpsPython --default-branch-only --fork-name "$repositoryName" --clone

# Change directory to the local repository
cd "$repositoryName"

# Remove the upstream remote and set the upstream to the main branch
git remote remove upstream
git push --set-upstream origin main

# Retrieve the repository full name (org/repo)
repositoryFullName=$(gh repo view --json nameWithOwner -q ".nameWithOwner")

# Set the default repository
gh repo set-default "https://github.com/${repositoryFullName}"

# Create environment
gh api --method PUT -H "Accept: application/vnd.github+json" "repos/${repositoryFullName}/environments/${environmentName}"

# Retrieve the current subscription and current tenant identifiers using Azure CLI
subscriptionId=$(az account show --query "id" -o tsv)
tenantId=$(az account show --query "tenantId" -o tsv)

# Create an App Registration and its associated service principal using Azure CLI
appId=$(az ad app create --display-name "GitHub Action OIDC for ${repositoryFullName}" --query "appId" -o tsv)
servicePrincipalId=$(az ad sp create --id "$appId" --query "id" -o tsv)

# Assign the contributor role to the service principal on the subscription using Azure CLI
az role assignment create --role contributor --subscription "$subscriptionId" --assignee-object-id "$servicePrincipalId" --assignee-principal-type ServicePrincipal --scope "/subscriptions/$subscriptionId"

# Prepare parameters for federated credentials
parametersJson='{
"name": "'"$environmentName"'",
"issuer": "https://token.actions.githubusercontent.com",
"subject": "repo:'"$repositoryFullName"':environment:'"$environmentName"'",
"description": "Development",
"audiences": [
"api://AzureADTokenExchange"
]
}'

# Create federated credentials using Azure CLI
az ad app federated-credential create --id "$appId" --parameters "$parametersJson"

# Create GitHub secrets needed for the GitHub Actions using GitHub CLI
gh secret set AZURE_TENANT_ID --body "$tenantId" --env "$environmentName"
gh secret set AZURE_SUBSCRIPTION_ID --body "$subscriptionId" --env "$environmentName"
gh secret set AZURE_CLIENT_ID --body "$appId" --env "$environmentName"
gh secret set DOCKER_PASSWORD --body "robertcarry" --env "$environmentName"
gh secret set DOCKER_USENAME --body "dckr_pat_e2lZ9YgpMt8APE-Qxzn89u6mt28" --env "$environmentName"

# Run the GitHub workflow
gh workflow enable main.yml
gh workflow run main.yml

# Get the run ID
runId=$(gh run list --workflow=main.yml --json databaseId -q ".[0].databaseId")

# Watch the run
gh run watch "$runId"

# Open the repository in the browser
gh repo view -w
2 changes: 1 addition & 1 deletion documentation/step_0_setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ jobs:
run: |
pipenv install --dev
pipenv run flake8 .
' > python-ci.yml
' > python-main.yml
```

Now we can protect your "main" branch on github. Add a security constrain, that your code must pass the CI before merging.
Expand Down
63 changes: 10 additions & 53 deletions documentation/workshop_cloud_nord.md → documentation/workshop.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ You will contribute to the MLOpsPython project.

## 0. Prerequisite

- azure cli https://learn.microsoft.com/fr-fr/cli/azure/install-azure-cli
- github cli https://cli.github.com

- Pycharm (https://www.jetbrains.com/pycharm/)
- Download and install python 3.10.x on your laptop.
https://www.python.org/downloads/
Expand Down Expand Up @@ -52,64 +55,19 @@ https://www.microsoftazurepass.com/Home/HowTo?WT.mc_id=DOP-MVP-5003370

### 2.3. Fork the GitHub project MLOpsPython

Prerequisite to install on your laptop:

1. Fork https://github.com/guillaume-chervet/MLOpsPython repository
2. Go to GitHub Action Tab and activate it !
3. Clone your new own **MLOpsPython** repository

### 2.4. Set up Azure Secret for GitHub Action

https://learn.microsoft.com/en-us/azure/machine-learning/how-to-github-actions-machine-learning?tabs=userlevel#step-2-authenticate-with-azure?WT.mc_id=DOP-MVP-5003370

Download and install azure-cli from :

https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-windows?tabs=azure-cli&WT.mc_id=DOP-MVP-5003370

```bash
# Log to azure
az login

# az account show
# >> show your current account
# az account list
# >> list all your account

# set the subscription
az account set -s <subscription-id>

# Create a service principal with the az ad sp create-for-rbac command in the Azure CLI. Run this command with Azure Cloud Shell in the Azure portal or by selecting the Try it button.
```bash
az ad sp create-for-rbac --name "azure-admin" --role contributor \
--scopes /subscriptions/<subscription-id> \
--sdk-auth
```
In the example above, replace the placeholders with your subscription ID, resource group name, and app name. The output is a JSON object with the role assignment credentials that provide access to your App Service app similar to below. Copy this JSON object for later.

```json
{
"clientId": "<GUID>",
"clientSecret": "<GUID>",
"subscriptionId": "<GUID>",
"tenantId": "<GUID>",
(...)
}
```

Create secrets in Github Action

1. In GitHub, go to your repository.
2. Select Security > Secrets and variables > Actions.
3. Select New repository secret.
4. Paste the entire JSON output from the Azure CLI command into the secret's value field. Give the secret the name AZURE_CREDENTIALS.
Je vous mets à disposition les fichiers de script ci-dessous :
Copier un de ces deux fichiers disponibles ici https://github.com/guillaume-chervet/... chez vous et exécuter le Powershell (.ps1) si vous êtes sous Windows et le bash (.sh) si vous êtes sous linux ou mac.

Il se base sur ce super Article réalisé par Alexandre Nédélec que je remercie : https://www.techwatching.dev/posts/sc...

### 2.5. Set up Docker Secret for GitHub Action

Create or login to docker hub account: https://www.docker.com/

1. Generate a new access token (Read & Write): https://hub.docker.com/settings/security
2. Create a new secret in your GitHub repository :
- DOCKER_PASSWORD
- DOCKER_USERNAME

### 2.6. Adapt GitHub Action Parameters

Expand All @@ -119,10 +77,9 @@ Inside "./.github/workflows/ci.yml" file

env:
AZURE_RESOURCE_GROUP_NAME: "azure-ml-<your-name>"
AZURE_SUBSCRIPTION_ID: "<subscription-id>"
AZURE_ML_WORKSPACE_NAME: "cats-dogs-<your-name>"
AZURE_WEBAPP_NAME: "cats-dogs-<your-name>"
DOCKER_IMAGE_NAME: "<your-docker-login>/mlopspython"
DOCKER_IMAGE_NAME: "robertcarry/mlopspython-<your-name>"

````

Expand All @@ -142,7 +99,7 @@ This will trigger the GitHub Action.

## 3. Getting Started on your laptop

Follow "Get Started" section to run the projet on your laptop :
Follow "Get Started" section to run the project on your laptop :

https://github.com/your-github-login/MLOpsPython

Expand Down
2 changes: 1 addition & 1 deletion production/api/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ cachetools==5.3.1 ; python_version >= "3.10" and python_version < "3.11"
certifi==2023.7.22 ; python_version >= "3.10" and python_version < "3.11"
charset-normalizer==3.2.0 ; python_version >= "3.10" and python_version < "3.11"
click==8.1.7 ; python_version >= "3.10" and python_version < "3.11"
colorama==0.4.6; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows"
colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows"
exceptiongroup==1.1.3 ; python_version >= "3.10" and python_version < "3.11"
fastapi==0.95.0 ; python_version >= "3.10" and python_version < "3.11"
flatbuffers==23.5.26 ; python_version >= "3.10" and python_version < "3.11"
Expand Down
1 change: 1 addition & 0 deletions train/azureml_run_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def azureml_pipeline(pdfs_input_data: Input(type=URI_FOLDER),

ml_client.jobs.stream(pipeline_job.name)

credential.get_token("https://management.azure.com/.default")

integration_dataset = Data(
name="cats-dogs-others-extraction",
Expand Down
2 changes: 1 addition & 1 deletion train/extraction/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@

result = extract_images(pdfs_input, images_output)
mlflow.log_metric("number_files_input", result.number_files_input)
mlflow.log_metric("number_images_output", result.number_images_output)
mlflow.log_metric("number_images_output", result.number_images_output)

0 comments on commit 66e00cf

Please sign in to comment.