diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..11a87e98 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.envrc +.env.toml +.terraform.lock.* +.terraform diff --git a/k8s/02_efs.sh b/k8s/02_efs.sh index 421f4af9..d82a0174 100755 --- a/k8s/02_efs.sh +++ b/k8s/02_efs.sh @@ -37,8 +37,8 @@ fi echo "Detected Security Group ID: $securityGroupId" -subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | head -1` -subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | tail -1` +subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | head -1` +subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | tail -1` echo "Detected Subnet: $subnetIdZoneA" echo "Detected Subnet: $subnetIdZoneB" diff --git a/k8s/03_ebs.sh b/k8s/03_ebs.sh index b7456da0..27934fac 100755 --- a/k8s/03_ebs.sh +++ b/k8s/03_ebs.sh @@ -31,7 +31,7 @@ terraform init -backend-config=bucket=$S3_BUCKET \ terraform apply -var aws_region=$AWS_REGION -var aws_availability_zone=${AWS_REGION}a -auto-approve -export TG_EBS_DATADIR_VOLUME_ID="aws://`terraform output availability_zone`/`terraform output volume_id`" +export TG_EBS_DATADIR_VOLUME_ID="aws://`terraform output --raw availability_zone`/`terraform output --raw volume_id`" popd diff --git a/k8s/cluster.yaml b/k8s/cluster.yaml index 0c1b8b47..43fd8e95 100644 --- a/k8s/cluster.yaml +++ b/k8s/cluster.yaml @@ -36,13 +36,13 @@ spec: etcdClusters: - cpuRequest: 200m etcdMembers: - - instanceGroup: master-${ZONE} + - instanceGroup: master-${AWS_REGION} name: a memoryRequest: 100Mi name: main - cpuRequest: 100m etcdMembers: - - instanceGroup: master-${ZONE} + - instanceGroup: master-${AWS_REGION} name: a memoryRequest: 100Mi name: events @@ -118,7 +118,7 @@ metadata: creationTimestamp: null labels: kops.k8s.io/cluster: ${CLUSTER_NAME} - name: master-${ZONE} + name: master-${AWS_REGION} spec: additionalUserData: - name: myscript.sh diff --git a/k8s/delete_efs.sh b/k8s/delete_efs.sh index 3322f4c0..b643888f 100755 --- a/k8s/delete_efs.sh +++ b/k8s/delete_efs.sh @@ -29,8 +29,8 @@ fi echo "Detected Security Group ID: $securityGroupId" -subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | head -1` -subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | tail -1` +subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | head -1` +subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | tail -1` echo "Detected Subnet: $subnetIdZoneA" echo "Detected Subnet: $subnetIdZoneB" diff --git a/k8s/ebs-terraform/ebs.tf b/k8s/ebs-terraform/ebs.tf index c47f074f..132a3aa5 100644 --- a/k8s/ebs-terraform/ebs.tf +++ b/k8s/ebs-terraform/ebs.tf @@ -1,6 +1,12 @@ -provider "aws" { - region = var.aws_region - version = "~> 2.50" +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } + + required_version = ">= 0.14" } # EBS for Testground daemon datadir diff --git a/k8s/ebs-terraform/variables.tf b/k8s/ebs-terraform/variables.tf index 5177a818..147786ca 100644 --- a/k8s/ebs-terraform/variables.tf +++ b/k8s/ebs-terraform/variables.tf @@ -3,10 +3,11 @@ variable "aws_region" {} variable "aws_availability_zone" {} variable "default_tags" { - type = "map" + type = map + # this should be dynamic default = { Name = "taas-daemon-datadir-volume" - KubernetesCluster = "anton-kops.k8s.local" + KubernetesCluster = "testground.k8s.local" } } diff --git a/k8s/efs-terraform/efs.tf b/k8s/efs-terraform/efs.tf index 2e73d34e..40605dcb 100644 --- a/k8s/efs-terraform/efs.tf +++ b/k8s/efs-terraform/efs.tf @@ -1,6 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } + + required_version = ">= 0.14" +} + provider "aws" { region = var.aws_region - version = "~> 2.50" } # EFS for Testground outputs diff --git a/k8s/testground-daemon/config-map-env-toml.yml b/k8s/testground-daemon/config-map-env-toml.yml index cd46a94d..69d5c23a 100644 --- a/k8s/testground-daemon/config-map-env-toml.yml +++ b/k8s/testground-daemon/config-map-env-toml.yml @@ -6,7 +6,7 @@ metadata: data: .env.toml: | ["aws"] - region = "eu-west-2" + region = "us-east-1" [runners."cluster:k8s"] run_timeout_min = 15 diff --git a/k8s/testground-daemon/deployment.yml b/k8s/testground-daemon/deployment.yml index 799a1a3d..be6698ee 100644 --- a/k8s/testground-daemon/deployment.yml +++ b/k8s/testground-daemon/deployment.yml @@ -20,20 +20,21 @@ spec: dnsPolicy: ClusterFirstWithHostNet nodeSelector: testground.node.role.infra: "true" - initContainers: - - name: iproute-add - image: busybox:1.31.1 - securityContext: - privileged: true - command: - - sh - - -ac - - > - while [ "$GW" = "" ]; do export GW=$(ip route | grep cni0 | awk '{print $7}'); echo "Got GW: $GW"; sleep 5; done; - echo $GW && - ip route && - ip route add 100.64.0.0/16 via $GW && - ip route || true; + # Why this? it doesn't work + # initContainers: + # - name: iproute-add + # image: busybox:1.31.1 + # securityContext: + # privileged: true + # command: + # - sh + # - -ac + # - > + # while [ "$GW" = "" ]; do export GW=$(ip route | grep cni0 | awk '{print $7}'); echo "Got GW: $GW"; sleep 5; done; + # echo $GW && + # ip route && + # ip route add 100.64.0.0/16 via $GW && + # ip route || true; containers: - name: goproxy image: iptestground/goproxy:2.0.2 @@ -44,8 +45,8 @@ spec: - name: envtoml mountPath: /root/testground/.env.toml subPath: .env.toml - - name: efs-pvc - mountPath: "/go" + # - name: efs-pvc + # mountPath: "/go" resources: requests: memory: 512Mi @@ -66,8 +67,8 @@ spec: volumeMounts: - name: daemon-datadir mountPath: "/root/testground/" - - name: efs-pvc - mountPath: "/efs" + # - name: efs-pvc + # mountPath: "/efs" - name: dockersock mountPath: "/var/run/docker.sock" - name: envtoml @@ -80,9 +81,9 @@ spec: limits: memory: 2048Mi volumes: - - name: efs-pvc - persistentVolumeClaim: - claimName: efs + # - name: efs-pvc + # persistentVolumeClaim: + # claimName: efs - name: daemon-datadir persistentVolumeClaim: claimName: testground-daemon-datadir-pvc @@ -92,3 +93,17 @@ spec: - name: envtoml configMap: name: env-toml-cfg + +# missing policy +# { +# "Sid": "kopsK8sEC2NodePerms", +# "Effect": "Allow", +# "Action": [ +# "ec2:DescribeInstances", +# "ec2:AttachVolume", +# "ec2:DetachVolume" +# ], +# "Resource": [ +# "*" +# ] +# }, diff --git a/k8s/testground-infra/requirements.lock b/k8s/testground-infra/requirements.lock index 294a3aaf..e981122d 100644 --- a/k8s/testground-infra/requirements.lock +++ b/k8s/testground-infra/requirements.lock @@ -1,9 +1,9 @@ dependencies: - name: redis repository: https://charts.bitnami.com/bitnami - version: 10.6.6 + version: 15.6.4 - name: testground-dashboards repository: "" version: 0.0.* -digest: sha256:2a9e152a6d30e2aca28f44c2e5131469c014b505888c5e3a6eeac2af94ba0915 -generated: "2020-04-14T14:57:04.250358476-07:00" +digest: sha256:2879642640ddf1ee3f5e19960dbac10dcdb9760a04a1b5dc6941e59dff566aff +generated: "2021-12-14T12:48:36.199922+01:00" diff --git a/k8s/testground-infra/values.yaml b/k8s/testground-infra/values.yaml index ca21b4db..d5aba3ae 100644 --- a/k8s/testground-infra/values.yaml +++ b/k8s/testground-infra/values.yaml @@ -11,18 +11,23 @@ redis: metrics: enabled: true - serviceMonitor: - enabled: true - namespace: default + # This doesn work: Error: INSTALLATION FAILED: unable to build kubernetes objects from release manifest: unable to recognize "": no matches for kind "ServiceMonitor" + # This needs to be installed with: helm install testground-infra . + # serviceMonitor: + # enabled: true + # namespace: default resources: requests: memory: 256Mi cpu: 200m limits: memory: 256Mi + architecture: standalone cluster: enabled: false usePassword: false + auth: + enabled: false securityContext: sysctls: - name: net.core.somaxconn