Skip to content

Commit

Permalink
fixed it on develop
Browse files Browse the repository at this point in the history
  • Loading branch information
bcdurak committed May 7, 2024
2 parents 05c1ede + cc15953 commit 12c5305
Show file tree
Hide file tree
Showing 331 changed files with 2,344 additions and 235 deletions.
103 changes: 103 additions & 0 deletions .github/workflows/aws-integration-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
permissions:
contents: read
name: LocalStack AWS Integration Test
on:
workflow_call:
workflow_dispatch:
jobs:
aws_modular_integration_test:
name: aws_modular_integration_test
runs-on: ubuntu-latest
services:
setup-localstack-service:
image: localstack/localstack
ports:
- '4566:4566'
env:
SERVICES: 's3,iam,sts'
DEFAULT_REGION: eu-north-1
FORCE_NONINTERACTIVE: 1
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
steps:
- name: Checkout Repository
uses: actions/[email protected]

- name: Setup Terraform
uses: hashicorp/[email protected]
with:
terraform_version: 1.6.0

- name: Copy Override File
run: |
cp tests/integration/_override.tf src/mlstacks/terraform/aws-modular/_override.tf
- name: Apply Terraform Configuration
run: |
export TF_CLI_ARGS_apply="-compact-warnings"
terraform init -backend-config="path=./terraform.tfstate"
terraform validate
terraform apply -auto-approve -var-file="../../../../tests/integration/aws-modular/local.tfvars"
working-directory: src/mlstacks/terraform/aws-modular

- name: Refresh Terraform State
run: terraform refresh
working-directory: src/mlstacks/terraform/aws-modular

- name: Output Stack YAML Path
id: set_output
run: |
OUTPUT=$(terraform-bin output -raw stack-yaml-path)
echo "stack_yaml_path=$OUTPUT" >> $GITHUB_OUTPUT
working-directory: src/mlstacks/terraform/aws-modular
env:
terraform_wrapper: false

- name: Run Tests to Verify Resource Provisioning
run: |
STACK_YAML_PATH="${{ steps.set_output.outputs.stack_yaml_path }}"
ABSOLUTE_PATH="${GITHUB_WORKSPACE}/src/mlstacks/terraform/aws-modular/${STACK_YAML_PATH}"
../../../../tests/integration/aws-modular/verify_stack.sh "$ABSOLUTE_PATH"
working-directory: src/mlstacks/terraform/aws-modular

aws_remote_state_integration_test:
name: aws_remote_state_integration_test
runs-on: ubuntu-latest
services:
setup-localstack-service:
image: localstack/localstack
ports:
- '4566:4566'
env:
SERVICES: 's3,dynamodb,iam,sts'
DEFAULT_REGION: eu-north-1
FORCE_NONINTERACTIVE: 1
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
steps:
- name: Checkout Repository
uses: actions/[email protected]

- name: Setup Terraform
uses: hashicorp/[email protected]
with:
terraform_version: 1.6.0

- name: Copy Override File
run: |
cp tests/integration/_override.tf src/mlstacks/terraform/aws-remote-state/_override.tf
- name: Apply Terraform Configuration for aws-remote-state
run: |
export TF_CLI_ARGS_apply="-compact-warnings"
cd src/mlstacks/terraform/aws-remote-state
terraform init -backend-config="path=./terraform.tfstate"
terraform validate
terraform apply -auto-approve -var-file="../../../../tests/integration/aws-remote-state/local.tfvars"
- name: Run Tests to Verify Resource Provisioning
run: ./tests/integration/aws-remote-state/verify_stack.sh
env:
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_DEFAULT_REGION: eu-north-1
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@ jobs:
files: "."
config: ./.typos.toml

localstack-aws-integration-test:
uses: ./.github/workflows/aws-integration-test.yml
secrets: inherit

aws_test:
name: aws_test
runs-on: ubuntu-latest
Expand Down
7 changes: 7 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,13 @@ if any existing recipes cover their creation.
```
git checkout -b <new-branch-name> develop
```
or, if that doesn't work:
```
git checkout -b <new-branch-name> origin/develop
```

This alternative can be useful in cases where your local develop branch is out of sync with the remote repository, or if you want to ensure that your new branch is based on the most recent changes from the remote repository.

2. Fetch the latest changes from the remote `develop` branch:
```
git fetch origin develop
Expand Down
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/aws_account.tf
Original file line number Diff line number Diff line change
@@ -1,2 +1,6 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# fetch the current account ID for use in container registry URL
data "aws_caller_identity" "current" {}
data "aws_caller_identity" "current" {}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/configure_docker.tf
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up local docker client to access the newly created registry
resource "null_resource" "configure-local-docker" {
provisioner "local-exec" {
command = "aws ecr get-login-password --region ${local.region} | docker login --username AWS --password-stdin ${data.aws_caller_identity.current.account_id}.dkr.ecr.${local.region}.amazonaws.com"
}

}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/configure_kubectl.tf
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up local kubectl client to access the newly created cluster
resource "null_resource" "configure-local-kubectl" {
provisioner "local-exec" {
command = "aws eks --region ${local.region} update-kubeconfig --name ${data.aws_eks_cluster.cluster.name} --alias terraform"
}
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/ecr.tf
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# add an optional container registry
resource "aws_ecr_repository" "zenml-ecr-repository" {
name = local.ecr.name
image_tag_mutability = "MUTABLE"
count = local.ecr.enable_container_registry ? 1 : 0
tags = local.tags
}
}
4 changes: 4 additions & 0 deletions aws-kubeflow-kserve/eks.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# eks module to create a cluster
# newer versions of it had some error so going with v17.23.0 for now
module "eks" {
Expand Down
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/get_URIs.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# get URI for MLflow tracking server
data "kubernetes_service" "mlflow_tracking" {
metadata {
Expand All @@ -18,4 +22,4 @@ data "kubernetes_service" "kserve_ingress" {
depends_on = [
module.kserve
]
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/helm.tf
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# A default (non-aliased) provider configuration for "helm"
provider "helm" {
kubernetes {
host = data.aws_eks_cluster.cluster.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
token = data.aws_eks_cluster_auth.cluster.token
}
}
}
4 changes: 4 additions & 0 deletions aws-kubeflow-kserve/kserve-module/cert_manager.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# create a cert-manager release
resource "helm_release" "cert-manager" {
name = "cert-manager"
Expand Down
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/kserve-module/istio.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up istio for kserve
resource "null_resource" "create-istio-kserve" {
provisioner "local-exec" {
Expand Down Expand Up @@ -32,4 +36,4 @@ resource "null_resource" "create-istio-kserve" {
depends_on = [
null_resource.create-knative-serving,
]
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/kserve-module/knative_serving.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up istio for kserve
resource "null_resource" "create-knative-serving" {
provisioner "local-exec" {
Expand All @@ -17,4 +21,4 @@ resource "null_resource" "create-knative-serving" {
command = "kubectl delete -f https://github.com/knative/serving/releases/download/knative-v1.6.0/serving-core.yaml"
}

}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/kserve-module/kserve.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up istio for kserve
resource "null_resource" "kserve" {
provisioner "local-exec" {
Expand Down Expand Up @@ -26,4 +30,4 @@ resource "kubernetes_namespace" "workloads" {
metadata {
name = var.workloads_namespace
}
}
}
4 changes: 4 additions & 0 deletions aws-kubeflow-kserve/kserve-module/providers.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# defining the providers required by the mlflow module
terraform {
required_providers {
Expand Down
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/kserve-module/variables.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# namespace to create inference services in
variable "workloads_namespace" {
type = string
default = "zenml-workloads"
}
}
4 changes: 4 additions & 0 deletions aws-kubeflow-kserve/kserve.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# create kserve module
module "kserve" {
source = "./kserve-module"
Expand Down
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/kubeflow.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up kubeflow
resource "null_resource" "kubeflow" {
provisioner "local-exec" {
Expand Down Expand Up @@ -30,4 +34,4 @@ resource "null_resource" "kubeflow" {
null_resource.configure-local-kubectl,
module.eks,
]
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/kubernetes.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# a default (non-aliased) provider configuration for "kubernetes"
# not defining the kubernetes provider throws an error while running the eks module
provider "kubernetes" {
Expand All @@ -10,4 +14,4 @@ provider "kubectl" {
host = data.aws_eks_cluster.cluster.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
token = data.aws_eks_cluster_auth.cluster.token
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/locals.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# config values to use across the module
locals {
prefix = "kflow"
Expand Down Expand Up @@ -39,4 +43,4 @@ locals {
"managedBy" = "terraform"
"application" = local.prefix
}
}
}
3 changes: 3 additions & 0 deletions aws-kubeflow-kserve/mlflow-module/README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# MLflow Terraform Module

**Deprecation warning:** this code and documentation has been deprecated. Please
visit the corresponding location from `src/mlstacks/terraform` for the latest code and documentation.

## Input Variables

Input | Description
Expand Down
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/mlflow-module/ingress.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# set up the nginx ingress controller and the ingress with basic auth

resource "kubernetes_namespace" "nginx-ns" {
Expand Down Expand Up @@ -43,4 +47,4 @@ resource "kubernetes_ingress_v1" "mlflow-ingress" {
}
}
}
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/mlflow-module/mlflow.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

# create the mlflow tracking server deployment
resource "helm_release" "mlflow-tracking" {

Expand Down Expand Up @@ -63,4 +67,4 @@ resource "helm_release" "mlflow-tracking" {
name = "artifactRoot.gcs.bucket"
value = var.artifact_GCS_Bucket
}
}
}
6 changes: 5 additions & 1 deletion aws-kubeflow-kserve/mlflow-module/output.tf
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# DEPRECATION WARNING: This code has been deprecated
# The maintained & current code can be found at src/mlstacks/terraform/
# under the same relative location.

output "ingress-controller-name" {
value = helm_release.nginx-controller.name
}
output "ingress-controller-namespace" {
value = kubernetes_namespace.nginx-ns.metadata[0].name
}
}
Loading

0 comments on commit 12c5305

Please sign in to comment.