Skip to content

Commit

Permalink
neonvm: add arm multus and whereabouts manifests
Browse files Browse the repository at this point in the history
  • Loading branch information
mikhail-sakhnov committed Oct 28, 2024
1 parent 8a7a20c commit bd97eaf
Show file tree
Hide file tree
Showing 14 changed files with 273 additions and 8 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/e2e-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,9 @@ jobs:
run: |
rendered () { echo "rendered_manifests/$1"; }
kubectl apply -f $(rendered multus.yaml)
kubectl apply -f $(rendered multus-amd64.yaml)
kubectl -n kube-system rollout status daemonset kube-multus-ds
kubectl apply -f $(rendered whereabouts.yaml)
kubectl apply -f $(rendered whereabouts-amd64.yaml)
kubectl -n kube-system rollout status daemonset whereabouts
kubectl apply -f $(rendered neonvm-runner-image-loader.yaml)
kubectl -n neonvm-system rollout status daemonset neonvm-runner-image-loader
Expand Down
18 changes: 12 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -316,10 +316,12 @@ render-manifests: $(RENDERED) kustomize
cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force
cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force
# Build:
$(KUSTOMIZE) build neonvm/config/whereabouts > $(RENDERED)/whereabouts.yaml
$(KUSTOMIZE) build neonvm/config/whereabouts-amd64 > $(RENDERED)/whereabouts-amd64.yaml
$(KUSTOMIZE) build neonvm/config/whereabouts-arm64 > $(RENDERED)/whereabouts-arm64.yaml
$(KUSTOMIZE) build neonvm/config/multus-aks > $(RENDERED)/multus-aks.yaml
$(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml
$(KUSTOMIZE) build neonvm/config/multus > $(RENDERED)/multus.yaml
$(KUSTOMIZE) build neonvm/config/multus-amd64 > $(RENDERED)/multus-amd64.yaml
$(KUSTOMIZE) build neonvm/config/multus-arm64 > $(RENDERED)/multus-arm64.yaml
$(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml
$(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml
$(KUSTOMIZE) build neonvm-vxlan-controller > $(RENDERED)/neonvm-vxlan-controller.yaml
Expand All @@ -341,10 +343,14 @@ render-release: $(RENDERED) kustomize
cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER)
cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT)
# Build:
$(KUSTOMIZE) build neonvm/config/whereabouts > $(RENDERED)/whereabouts.yaml
$(KUSTOMIZE) build neonvm/config/whereabouts-amd64 > $(RENDERED)/whereabouts-amd64.yaml
# TODO: I ain't sure if we need arm64 for render-release target
# $(KUSTOMIZE) build neonvm/config/whereabouts-arm64 > $(RENDERED)/whereabouts-arm64.yaml
$(KUSTOMIZE) build neonvm/config/multus-aks > $(RENDERED)/multus-aks.yaml
$(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml
$(KUSTOMIZE) build neonvm/config/multus > $(RENDERED)/multus.yaml
$(KUSTOMIZE) build neonvm/config/multus-amd64 > $(RENDERED)/multus-amd64.yaml
# TODO: I ain't sure if we need arm64 for render-release target
$(KUSTOMIZE) build neonvm/config/multus-arm64 > $(RENDERED)/multus-arm64.yaml
$(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml
$(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml
$(KUSTOMIZE) build neonvm-vxlan-controller > $(RENDERED)/neonvm-vxlan-controller.yaml
Expand All @@ -360,9 +366,9 @@ render-release: $(RENDERED) kustomize

.PHONY: deploy
deploy: check-local-context docker-build load-images render-manifests kubectl ## Deploy controller to the K8s cluster specified in ~/.kube/config.
$(KUBECTL) apply -f $(RENDERED)/multus.yaml
$(KUBECTL) apply -f $(RENDERED)/multus-$(TARGET_ARCH).yaml
$(KUBECTL) -n kube-system rollout status daemonset kube-multus-ds
$(KUBECTL) apply -f $(RENDERED)/whereabouts.yaml
$(KUBECTL) apply -f $(RENDERED)/whereabouts-$(TARGET_ARCH).yaml
$(KUBECTL) -n kube-system rollout status daemonset whereabouts
$(KUBECTL) apply -f $(RENDERED)/neonvm-runner-image-loader.yaml
$(KUBECTL) -n neonvm-system rollout status daemonset neonvm-runner-image-loader
Expand Down
1 change: 1 addition & 0 deletions neonvm-vxlan-controller/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ spec:
operator: In
values:
- amd64
- arm64
- key: kubernetes.io/os
operator: In
values:
Expand Down
File renamed without changes.
File renamed without changes.
2 changes: 2 additions & 0 deletions neonvm/config/multus-arm64/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
This deployment is supposed to be used only on local cluster.
It is different from multus-local-amd64 and uses multus 4.X due to image availability for proper architecture.
31 changes: 31 additions & 0 deletions neonvm/config/multus-arm64/cluster_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: multus
rules:
- apiGroups:
- k8s.cni.cncf.io
resources:
- '*'
verbs:
- '*'
- apiGroups:
- ""
resources:
- pods
- pods/status
verbs:
- get
- update
- watch
- list
- apiGroups:
- ""
- events.k8s.io
resources:
- events
verbs:
- create
- patch
- update
21 changes: 21 additions & 0 deletions neonvm/config/multus-arm64/config_map.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
---
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-daemon-config
namespace: kube-system
labels:
tier: node
app: multus
data:
daemon-config.json: |
{
"chrootDir": "/hostroot",
"cniVersion": "0.3.1",
"logLevel": "verbose",
"logToStderr": true,
"cniConfigDir": "/host/etc/cni/net.d",
"multusAutoconfigDir": "/host/etc/cni/net.d",
"multusConfigFile": "auto",
"socketDir": "/host/run/multus/"
}
137 changes: 137 additions & 0 deletions neonvm/config/multus-arm64/daemonset_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: multus
name: multus
tier: node
name: kube-multus-ds
namespace: kube-system
spec:
selector:
matchLabels:
name: multus
template:
metadata:
labels:
app: multus
name: multus
tier: node
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- arm64
- key: kubernetes.io/os
operator: In
values:
- linux
containers:
- command:
- /usr/src/multus-cni/bin/multus-daemon
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.1.3-thick
name: kube-multus
resources:
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 100m
memory: 50Mi
securityContext:
privileged: true
volumeMounts:
- name: cni
mountPath: /host/etc/cni/net.d
# multus-daemon expects that cnibin path must be identical between pod and container host.
# e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
# not to any other directory, like '/opt/bin' or '/usr/bin'.
- name: cnibin
mountPath: /opt/cni/bin
- name: host-run
mountPath: /host/run
- name: host-var-lib-cni-multus
mountPath: /var/lib/cni/multus
- name: host-var-lib-kubelet
mountPath: /var/lib/kubelet
mountPropagation: HostToContainer
- name: host-run-k8s-cni-cncf-io
mountPath: /run/k8s.cni.cncf.io
- name: host-run-netns
mountPath: /run/netns
mountPropagation: HostToContainer
- name: multus-daemon-config
mountPath: /etc/cni/net.d/multus.d
readOnly: true
- name: hostroot
mountPath: /hostroot
mountPropagation: HostToContainer
env:
- name: MULTUS_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
hostNetwork: true
hostPID: true
initContainers:
- name: install-multus-binary
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.1.3-thick
command:
- "cp"
- "/usr/src/multus-cni/bin/multus-shim"
- "/host/opt/cni/bin/multus-shim"
resources:
requests:
cpu: "10m"
memory: "15Mi"
securityContext:
privileged: true
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- name: cnibin
mountPath: /host/opt/cni/bin
mountPropagation: Bidirectional
serviceAccountName: multus
terminationGracePeriodSeconds: 10
tolerations:
- effect: NoSchedule
operator: Exists
volumes:
- hostPath:
path: /etc/cni/net.d
name: cni
- hostPath:
path: /opt/cni/bin
name: cnibin
- name: hostroot
hostPath:
path: /
- name: multus-daemon-config
configMap:
name: multus-daemon-config
items:
- key: daemon-config.json
path: daemon-config.json
- name: host-run
hostPath:
path: /run
- name: host-var-lib-cni-multus
hostPath:
path: /var/lib/cni/multus
- name: host-var-lib-kubelet
hostPath:
path: /var/lib/kubelet
- name: host-run-k8s-cni-cncf-io
hostPath:
path: /run/k8s.cni.cncf.io
- name: host-run-netns
hostPath:
path: /run/netns/

updateStrategy:
type: RollingUpdate
28 changes: 28 additions & 0 deletions neonvm/config/multus-arm64/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

bases:
- ../multus-common

images:
- name: kube-multus
newName: ghcr.io/k8snetworkplumbingwg/multus-cni
newTag: v4.1.3-thick

resources:
- config_map.yaml
patchesStrategicMerge:
- cluster_role.yaml
- daemonset_patch.yaml

patches:
- target:
kind: DaemonSet
name: kube-multus-ds
patch: |-
- op: replace
path: /spec/template/spec/containers/0/command
value: ["/usr/src/multus-cni/bin/multus-daemon"]
- op: replace
path: /spec/template/spec/containers/0/args
value: []
24 changes: 24 additions & 0 deletions neonvm/config/whereabouts-arm64/daemonset_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# patch the DaemonSet so that it's only running on nodes that we'd support
#
# The image we're is a linux amd64 image; it doesn't work on ARM or non-Linux.
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: whereabouts
namespace: kube-system
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- arm64
- key: kubernetes.io/os
operator: In
values:
- linux
15 changes: 15 additions & 0 deletions neonvm/config/whereabouts-arm64/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

bases:
# whereabouts CNI
- https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/v0.6.2/doc/crds/whereabouts.cni.cncf.io_ippools.yaml
- https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/v0.6.2/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml
- https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/v0.6.2/doc/crds/daemonset-install.yaml

patchesStrategicMerge:
- daemonset_patch.yaml

images:
- name: ghcr.io/k8snetworkplumbingwg/whereabouts
newTag: v0.6.2-arm64

0 comments on commit bd97eaf

Please sign in to comment.