[WIP] feat: port the NodeUnschedulable in-tree plugin to wasm

Makefile

@@ -5,13 +5,16 @@ golangci_lint := github.com/golangci/golangci-lint/cmd/[email protected]
 examples/advanced/main.wasm: examples/advanced/main.go
 	@(cd $(@D); tinygo build -o main.wasm -gc=custom -tags=custommalloc -scheduler=none --no-debug -target=wasi .)
 
+examples/nodeunschedulable/main.wasm: examples/nodeunschedulable/main.go
+	@(cd $(@D); tinygo build -o main.wasm -gc=custom -tags=custommalloc -scheduler=none --no-debug -target=wasi .)
+
 %/main.wasm: %/main.go
 	@(cd $(@D); tinygo build -o main.wasm -scheduler=none --no-debug -target=wasi .)
 
 .PHONY: build-tinygo
-build-tinygo: examples/nodenumber/main.wasm examples/advanced/main.wasm examples/imagelocality/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm \
+build-tinygo: examples/nodenumber/main.wasm examples/advanced/main.wasm examples/imagelocality/main.wasm examples/nodeunschedulable/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm \
 			  guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm guest/testdata/permit/main.wasm \
-			  internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm 
+			  internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm internal/e2e/scheduler_perf/wasm/nodeunschedulable/main.wasm
 
 %/main-debug.wasm: %/main.go
 	@(cd $(@D); tinygo build -o main-debug.wasm -gc=custom -tags=custommalloc -scheduler=none -target=wasi .)

examples/nodeunschedulable/go.mod

@@ -0,0 +1,19 @@
+module sigs.k8s.io/kube-scheduler-wasm-extension/examples/nodeunschedulable
+
+go 1.20
+
+replace sigs.k8s.io/kube-scheduler-wasm-extension/guest => ./../../guest
+
+replace sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto => ./../../kubernetes/proto
+
+require (
+	github.com/wasilibs/nottinygc v0.7.1
+	sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000
+	sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000
+)
+
+require (
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/magefile/mage v1.14.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)

examples/nodeunschedulable/go.sum

@@ -0,0 +1,8 @@
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
+github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/wasilibs/nottinygc v0.7.1 h1:rKu19+SFniRNuSo5NX7/wxpSpXmMUmkcyt/YiWLJg8w=
+github.com/wasilibs/nottinygc v0.7.1/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=

examples/nodeunschedulable/main.go

@@ -0,0 +1,84 @@
+/*
+   Copyright 2023 The Kubernetes Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Package main is the entrypoint of the %.wasm file, compiled with
+// '-target=wasi'. See /guest/RATIONALE.md for details.
+package main
+
+import (
+	_ "github.com/wasilibs/nottinygc"
+
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/helper"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/config"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog"
+	klogapi "sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog/api"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/plugin"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/util"
+	k8sproto "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api"
+)
+
+const (
+	// ErrReasonUnknownCondition is used for NodeUnknownCondition predicate error.
+	ErrReasonUnknownCondition = "node(s) had unknown conditions"
+	// ErrReasonUnschedulable is used for NodeUnschedulable predicate error.
+	ErrReasonUnschedulable = "node(s) were unschedulable"
+)
+
+// main is compiled to a WebAssembly function named "_start", called by the
+// wasm scheduler plugin during initialization.
+func main() {
+	p, err := New(klog.Get(), config.Get())
+	if err != nil {
+		panic(err)
+	}
+	plugin.Set(p)
+}
+
+func New(klog klogapi.Klog, jsonConfig []byte) (api.Plugin, error) {
+	return &NodeUnschedulable{}, nil
+}
+
+// NodeUnschedulable plugin filters nodes that set node.Spec.Unschedulable=true unless
+// the pod tolerates {key=node.kubernetes.io/unschedulable, effect:NoSchedule} taint.
+type NodeUnschedulable struct{}
+
+func (p *NodeUnschedulable) Name() string {
+	return "NodeUnschedulable"
+}
+
+func (p *NodeUnschedulable) Filter(_ api.CycleState, pod proto.Pod, nodeInfo api.NodeInfo) *api.Status {
+	node := nodeInfo.Node()
+	if node == nil {
+		return &api.Status{
+			Code:   api.StatusCodeUnschedulableAndUnresolvable,
+			Reason: ErrReasonUnknownCondition,
+		}
+	}
+	// If pod tolerate unschedulable taint, it's also tolerate `node.Spec.Unschedulable`.
+	podToleratesUnschedulable := helper.TolerationsTolerateTaint(pod.Spec().Tolerations, &k8sproto.Taint{
+		Key:    util.To(api.TaintNodeUnschedulable),
+		Effect: util.To(api.TaintEffectNoSchedule),
+	})
+	if *node.Spec().Unschedulable && !podToleratesUnschedulable {
+		return &api.Status{
+			Code:   api.StatusCodeUnschedulableAndUnresolvable,
+			Reason: ErrReasonUnschedulable,
+		}
+	}
+	return nil
+}

guest/api/framework.go

@@ -1,5 +1,16 @@
 package api
 
+type QueueingHint int
+
+const (
+	// QueueSkip implies that the cluster event has no impact on
+	// scheduling of the pod.
+	QueueSkip QueueingHint = iota
+
+	// Queue implies that the Pod may be schedulable by the event.
+	Queue
+)
+
 // ImageStateSummary provides summarized information about the state of an image.
 type ImageStateSummary struct {
 	// Size of the image

guest/api/helper/helpers.go

@@ -0,0 +1,16 @@
+package helper
+
+import (
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
+	protoapi "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api"
+)
+
+// TolerationsTolerateTaint checks if taint is tolerated by any of the tolerations.
+func TolerationsTolerateTaint(tolerations []*protoapi.Toleration, taint *protoapi.Taint) bool {
+	for i := range tolerations {
+		if api.ToleratesTaint(tolerations[i], taint) {
+			return true
+		}
+	}
+	return false
+}

guest/api/tolerations.go

@@ -0,0 +1,46 @@
+package api
+
+import v1 "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api"
+
+const (
+	// TaintNodeUnschedulable will be added when node becomes unschedulable
+	// and removed when node becomes schedulable.
+	TaintNodeUnschedulable string = "node.kubernetes.io/unschedulable"
+	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
+	// but allow all pods submitted to Kubelet without going through the scheduler
+	// to start, and allow all already-running pods to continue running.
+	// Enforced by the scheduler.
+	TaintEffectNoSchedule string = "NoSchedule"
+	TolerationOpExists    string = "Exists"
+	TolerationOpEqual     string = "Equal"
+)
+
+// ToleratesTaint checks if the toleration tolerates the taint.
+// The matching follows the rules below:
+//
+//  1. Empty toleration.effect means to match all taint effects,
+//     otherwise taint effect must equal to toleration.effect.
+//  2. If toleration.operator is 'Exists', it means to match all taint values.
+//  3. Empty toleration.key means to match all taint keys.
+//     If toleration.key is empty, toleration.operator must be 'Exists';
+//     this combination means to match all taint values and all taint keys.
+func ToleratesTaint(toleration *v1.Toleration, taint *v1.Taint) bool {
+	if len(toleration.GetEffect()) > 0 && toleration.Effect != taint.Effect {
+		return false
+	}
+
+	if len(toleration.GetKey()) > 0 && toleration.Key != taint.Key {
+		return false
+	}
+
+	// TODO: Use proper defaulting when Toleration becomes a field of PodSpec
+	switch *toleration.Operator {
+	// empty operator means Equal
+	case "", TolerationOpEqual:
+		return toleration.Value == taint.Value
+	case TolerationOpExists:
+		return true
+	default:
+		return false
+	}
+}

guest/util/utils.go

@@ -0,0 +1,33 @@
+package util
+
+import (
+	"fmt"
+)
+
+// As converts two objects to the given type.
+// Both objects must be of the same type. If not, an error is returned.
+// nil objects are allowed and will be converted to nil.
+func As[T any](oldObj, newobj interface{}) (T, T, error) {
+	var oldTyped T
+	var newTyped T
+	var ok bool
+	if newobj != nil {
+		newTyped, ok = newobj.(T)
+		if !ok {
+			return oldTyped, newTyped, fmt.Errorf("expected %T, but got %T", newTyped, newobj)
+		}
+	}
+
+	if oldObj != nil {
+		oldTyped, ok = oldObj.(T)
+		if !ok {
+			return oldTyped, newTyped, fmt.Errorf("expected %T, but got %T", oldTyped, oldObj)
+		}
+	}
+	return oldTyped, newTyped, nil
+}
+
+// To returns a pointer to the given value.
+func To[T any](v T) *T {
+	return &v
+}

internal/e2e/go.mod

@@ -17,6 +17,7 @@ require (
 	k8s.io/kube-scheduler v0.27.3
 	k8s.io/kubernetes v1.27.3
 	sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000
+	sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000
 	sigs.k8s.io/kube-scheduler-wasm-extension/scheduler v0.0.0-00010101000000-000000000000
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -192,7 +193,6 @@ require (
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
 

internal/e2e/scheduler_perf/config/sched-config-all-enabled.yaml

@@ -5,11 +5,11 @@ profiles:
       multiPoint:
         enabled:
           - name: wasm
-          - name: NodeNumber
+          - name: NodeUnschedulable
     pluginConfig:
       - name: wasm
         args:
-          guestURL: "file://../../../examples/nodenumber/main.wasm"
+          guestURL: "file://../../../examples/nodeunschedulable/main.wasm"
 extenders:
   - urlPrefix: "http://localhost:8080/"
     prioritizeVerb: "priorities"

internal/e2e/scheduler_perf/config/sched-config-plugin-only.yaml

@@ -4,4 +4,4 @@ profiles:
   - plugins:
       multiPoint:
         enabled:
-          - name: NodeNumber
+          - name: NodeUnschedulable

internal/e2e/scheduler_perf/config/sched-config-wasm-only.yaml

@@ -8,4 +8,4 @@ profiles:
     pluginConfig:
       - name: wasm
         args:
-          guestURL: "file://./wasm/nodenumber/main.wasm"
+          guestURL: "file://./wasm/nodeunschedulable/main.wasm"

internal/e2e/scheduler_perf/wasm/nodeunschedulable/README.md

@@ -0,0 +1,3 @@
+# NodeUnschedulable Plugin
+
+This is the nodeunschedulable wasm plugin based on the in-tree, which only implements Filter.

internal/e2e/scheduler_perf/wasm/nodeunschedulable/main.go

@@ -0,0 +1,84 @@
+/*
+   Copyright 2023 The Kubernetes Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Package main is the entrypoint of the %.wasm file, compiled with
+// '-target=wasi'. See /guest/RATIONALE.md for details.
+package main
+
+import (
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/helper"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/config"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog"
+	klogapi "sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog/api"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/plugin"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/util"
+	k8sproto "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api"
+)
+
+const (
+	// ErrReasonUnknownCondition is used for NodeUnknownCondition predicate error.
+	ErrReasonUnknownCondition = "node(s) had unknown conditions"
+	// ErrReasonUnschedulable is used for NodeUnschedulable predicate error.
+	ErrReasonUnschedulable = "node(s) were unschedulable"
+)
+
+func main() {
+	p, err := New(klog.Get(), config.Get())
+	if err != nil {
+		panic(err)
+	}
+	plugin.Set(p)
+}
+
+func New(klog klogapi.Klog, jsonConfig []byte) (api.Plugin, error) {
+	return &NodeUnschedulable{}, nil
+}
+
+// NodeUnschedulable plugin filters nodes that set node.Spec.Unschedulable=true unless
+// the pod tolerates {key=node.kubernetes.io/unschedulable, effect:NoSchedule} taint.
+type NodeUnschedulable struct{}
+
+func (p *NodeUnschedulable) Name() string {
+	return "NodeUnschedulable"
+}
+
+func (p *NodeUnschedulable) Filter(_ api.CycleState, pod proto.Pod, nodeInfo api.NodeInfo) *api.Status {
+	node := nodeInfo.Node()
+	if node == nil {
+		return &api.Status{
+			Code:   api.StatusCodeUnschedulableAndUnresolvable,
+			Reason: ErrReasonUnknownCondition,
+		}
+	}
+	// If pod tolerate unschedulable taint, it's also tolerate `node.Spec.Unschedulable`.
+	podToleratesUnschedulable := helper.TolerationsTolerateTaint(pod.Spec().Tolerations, &k8sproto.Taint{
+		Key:    util.To(api.TaintNodeUnschedulable),
+		Effect: util.To(api.TaintEffectNoSchedule),
+	})
+	unschedulable := false
+	if node.Spec().Unschedulable != nil {
+		unschedulable = *node.Spec().Unschedulable
+	}
+	if unschedulable && !podToleratesUnschedulable {
+		return &api.Status{
+			Code:   api.StatusCodeUnschedulableAndUnresolvable,
+			Reason: ErrReasonUnschedulable,
+		}
+	}
+	return nil
+}