diff --git a/Makefile b/Makefile index cf05cf1d..2642926d 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ examples/advanced/main.wasm: examples/advanced/main.go .PHONY: build-tinygo build-tinygo: examples/nodenumber/main.wasm examples/advanced/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm \ - guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm guest/testdata/permit/main.wasm + guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm guest/testdata/permit/main.wasm \ + internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm %/main-debug.wasm: %/main.go @(cd $(@D); tinygo build -o main-debug.wasm -gc=custom -tags=custommalloc -scheduler=none -target=wasi .) diff --git a/examples/advanced/main.wasm b/examples/advanced/main.wasm index 1455ce4b..1a8759c2 100755 Binary files a/examples/advanced/main.wasm and b/examples/advanced/main.wasm differ diff --git a/examples/nodenumber/main.wasm b/examples/nodenumber/main.wasm index 282d23d3..7485e7e2 100755 Binary files a/examples/nodenumber/main.wasm and b/examples/nodenumber/main.wasm differ diff --git a/guest/testdata/bind/main.wasm b/guest/testdata/bind/main.wasm index 74181685..ef1cf14b 100755 Binary files a/guest/testdata/bind/main.wasm and b/guest/testdata/bind/main.wasm differ diff --git a/guest/testdata/cyclestate/main.wasm b/guest/testdata/cyclestate/main.wasm index b4616062..3092050e 100755 Binary files a/guest/testdata/cyclestate/main.wasm and b/guest/testdata/cyclestate/main.wasm differ diff --git a/guest/testdata/filter/main.wasm b/guest/testdata/filter/main.wasm index b6be5043..f768f99c 100755 Binary files a/guest/testdata/filter/main.wasm and b/guest/testdata/filter/main.wasm differ diff --git a/guest/testdata/handle/main.wasm b/guest/testdata/handle/main.wasm index 169caebd..4c9a5e08 100755 Binary files a/guest/testdata/handle/main.wasm and b/guest/testdata/handle/main.wasm differ diff --git a/guest/testdata/reserve/main.wasm b/guest/testdata/reserve/main.wasm index 75b5366b..978e25d5 100755 Binary files a/guest/testdata/reserve/main.wasm and b/guest/testdata/reserve/main.wasm differ diff --git a/guest/testdata/score/main.wasm b/guest/testdata/score/main.wasm index 5d1f0821..1493b36d 100755 Binary files a/guest/testdata/score/main.wasm and b/guest/testdata/score/main.wasm differ diff --git a/internal/e2e/go.mod b/internal/e2e/go.mod index 7730cfdb..3e8ecbda 100644 --- a/internal/e2e/go.mod +++ b/internal/e2e/go.mod @@ -16,6 +16,7 @@ require ( k8s.io/klog/v2 v2.90.1 k8s.io/kube-scheduler v0.27.3 k8s.io/kubernetes v1.27.3 + sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000 sigs.k8s.io/kube-scheduler-wasm-extension/scheduler v0.0.0-00010101000000-000000000000 sigs.k8s.io/yaml v1.3.0 ) @@ -191,11 +192,12 @@ require ( k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) replace ( + sigs.k8s.io/kube-scheduler-wasm-extension/guest => ../../guest sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto => ../../kubernetes/proto - sigs.k8s.io/kube-scheduler-wasm-extension/scheduler => ../../scheduler ) diff --git a/internal/e2e/scheduler_perf/config/performance-config.yaml b/internal/e2e/scheduler_perf/config/performance-config.yaml index 9b154bcb..2d10e913 100644 --- a/internal/e2e/scheduler_perf/config/performance-config.yaml +++ b/internal/e2e/scheduler_perf/config/performance-config.yaml @@ -85,4 +85,4 @@ params: initNodes: 500 initPods: 500 - measurePods: 1000 + measurePods: 1000 \ No newline at end of file diff --git a/internal/e2e/scheduler_perf/config/sched-config-wasm-only.yaml b/internal/e2e/scheduler_perf/config/sched-config-wasm-only.yaml index 17c05465..917ea9ab 100644 --- a/internal/e2e/scheduler_perf/config/sched-config-wasm-only.yaml +++ b/internal/e2e/scheduler_perf/config/sched-config-wasm-only.yaml @@ -8,4 +8,4 @@ profiles: pluginConfig: - name: wasm args: - guestURL: "file://../../../examples/nodenumber/main.wasm" \ No newline at end of file + guestURL: "file://./wasm/nodenumber/main.wasm" \ No newline at end of file diff --git a/internal/e2e/scheduler_perf/wasm/nodenumber/README.md b/internal/e2e/scheduler_perf/wasm/nodenumber/README.md new file mode 100644 index 00000000..2f5f9b77 --- /dev/null +++ b/internal/e2e/scheduler_perf/wasm/nodenumber/README.md @@ -0,0 +1,4 @@ +# NodeNumber Plugin + +This is the nodenumber example wasm plugin, which only implements PreScore and Score. +It doesn't use any additional host functions (klog, handle, etc) so that scheduler_perf can measure the overhead truely. diff --git a/internal/e2e/scheduler_perf/wasm/nodenumber/main.go b/internal/e2e/scheduler_perf/wasm/nodenumber/main.go new file mode 100644 index 00000000..6a6ea96a --- /dev/null +++ b/internal/e2e/scheduler_perf/wasm/nodenumber/main.go @@ -0,0 +1,136 @@ +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package main is the entrypoint of the %.wasm file, compiled with +// '-target=wasi'. See /guest/RATIONALE.md for details. +package main + +import ( + "encoding/json" + "fmt" + + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/config" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog" + klogapi "sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog/api" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/plugin" +) + +// main is compiled to a WebAssembly function named "_start", called by the +// wasm scheduler plugin during initialization. +func main() { + p, err := New(klog.Get(), config.Get()) + if err != nil { + panic(err) + } + plugin.Set(p) +} + +func New(klog klogapi.Klog, jsonConfig []byte) (api.Plugin, error) { + var args nodeNumberArgs + if jsonConfig != nil { + if err := json.Unmarshal(jsonConfig, &args); err != nil { + panic(fmt.Errorf("decode arg into NodeNumberArgs: %w", err)) + } + klog.Info("NodeNumberArgs is successfully applied") + } + return &NodeNumber{reverse: args.Reverse}, nil +} + +// NodeNumber is an example plugin that favors nodes that share a numerical +// suffix with the pod name. +// +// For example, when a pod named "Pod1" is scheduled, a node named "Node1" gets +// a higher score than a node named "Node9". +// +// # Notes +// +// - Only the last character in names are considered. This means "Node99" is +// treated the same as "Node9" +// - The reverse field inverts the score. For example, when `reverse == true` +// a numeric match gets a results in a lower score than a match. +type NodeNumber struct { + reverse bool +} + +type nodeNumberArgs struct { + Reverse bool `json:"reverse"` +} + +const ( + // Name is the name of the plugin used in the plugin registry and configurations. + Name = "NodeNumber" + preScoreStateKey = "PreScore" + Name +) + +// preScoreState computed at PreScore and used at Score. +type preScoreState struct { + podSuffixNumber uint8 +} + +// EventsToRegister implements api.EnqueueExtensions +func (pl *NodeNumber) EventsToRegister() []api.ClusterEvent { + return []api.ClusterEvent{ + {Resource: api.Node, ActionType: api.Add}, + } +} + +// PreScore implements api.PreScorePlugin +func (pl *NodeNumber) PreScore(state api.CycleState, pod proto.Pod, _ proto.NodeList) *api.Status { + podnum, ok := lastNumber(pod.Spec().GetNodeName()) + if !ok { + return nil // return success even if its suffix is non-number. + } + state.Write(preScoreStateKey, &preScoreState{podSuffixNumber: podnum}) + return nil +} + +// Score implements api.ScorePlugin +func (pl *NodeNumber) Score(state api.CycleState, pod proto.Pod, nodeName string) (int32, *api.Status) { + var match bool + if data, ok := state.Read(preScoreStateKey); ok { + // Match is when there is a last digit, and it is the pod suffix. + nodenum, ok := lastNumber(nodeName) + match = ok && data.(*preScoreState).podSuffixNumber == nodenum + } else { + // Match is also when there is no pod spec node name. + match = true + } + + if pl.reverse { + match = !match // invert the condition. + } + + if match { + return 10, nil + } + return 0, nil +} + +// lastNumber returns the last number in the string or false. +func lastNumber(str string) (uint8, bool) { + if len(str) == 0 { + return 0, false + } + + // We have at least a single character name. See if the last is a digit. + lastChar := str[len(str)-1] + if '0' <= lastChar && lastChar <= '9' { + return lastChar - '0', true + } + return 0, false +} diff --git a/internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm b/internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm new file mode 100755 index 00000000..c33a73dd Binary files /dev/null and b/internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm differ diff --git a/scheduler/plugin/guest.go b/scheduler/plugin/guest.go index 9e14abb6..452be9ec 100644 --- a/scheduler/plugin/guest.go +++ b/scheduler/plugin/guest.go @@ -220,6 +220,11 @@ func (g *guest) normalizeScore(ctx context.Context) (framework.NodeScoreList, *f statusCode := int32(callStack[0]) statusReason := paramsFromContext(ctx).resultStatusReason normalizedScoreList := paramsFromContext(ctx).resultNormalizedScoreList + if len(normalizedScoreList) == 0 { + // Probably the guest didn't implement NormalizeScore(). + normalizedScoreList = paramsFromContext(ctx).nodeScoreList + } + return normalizedScoreList, framework.NewStatus(framework.Code(statusCode), statusReason) } diff --git a/scheduler/plugin/plugin_test.go b/scheduler/plugin/plugin_test.go index 85b6d218..5973e261 100644 --- a/scheduler/plugin/plugin_test.go +++ b/scheduler/plugin/plugin_test.go @@ -1380,8 +1380,8 @@ func TestPostBind(t *testing.T) { wasm error: unreachable wasm stack trace: - .runtime._panic(i32,i32) - .postbind() + main.runtime._panic(i32,i32) + main.postbind() >`, }, {