From 0f02c8352a63ce508dbed384bb4340d9b2b4c591 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Sat, 27 Mar 2021 12:28:16 +0000 Subject: [PATCH 01/20] Introduce builtin function go and makechan Builtin go(fn, arg1, arg2, ...) starts a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from the current running VM, and returns a job object that has wait, result, abort methods. Builtin makechan(size) accepts an optional size parameter, makes a channel to send/receive object, and returns a chan object that has send, recv, close methods. To be able to access *VM instance that it is running in for go() and makechan(), now VM will always pass VMObj to and only to BuiltinFunction as arg[0]. --- builtins.go | 194 +++++++++++++++++------------------------------ jobchan.go | 211 ++++++++++++++++++++++++++++++++++++++++++++++++++++ vm.go | 124 +++++++++++++++++++++++++++++- 3 files changed, 400 insertions(+), 129 deletions(-) create mode 100644 jobchan.go diff --git a/builtins.go b/builtins.go index b954d072..2e247390 100644 --- a/builtins.go +++ b/builtins.go @@ -1,130 +1,43 @@ package tengo -var builtinFuncs = []*BuiltinFunction{ - { - Name: "len", - Value: builtinLen, - }, - { - Name: "copy", - Value: builtinCopy, - }, - { - Name: "append", - Value: builtinAppend, - }, - { - Name: "delete", - Value: builtinDelete, - }, - { - Name: "splice", - Value: builtinSplice, - }, - { - Name: "string", - Value: builtinString, - }, - { - Name: "int", - Value: builtinInt, - }, - { - Name: "bool", - Value: builtinBool, - }, - { - Name: "float", - Value: builtinFloat, - }, - { - Name: "char", - Value: builtinChar, - }, - { - Name: "bytes", - Value: builtinBytes, - }, - { - Name: "time", - Value: builtinTime, - }, - { - Name: "is_int", - Value: builtinIsInt, - }, - { - Name: "is_float", - Value: builtinIsFloat, - }, - { - Name: "is_string", - Value: builtinIsString, - }, - { - Name: "is_bool", - Value: builtinIsBool, - }, - { - Name: "is_char", - Value: builtinIsChar, - }, - { - Name: "is_bytes", - Value: builtinIsBytes, - }, - { - Name: "is_array", - Value: builtinIsArray, - }, - { - Name: "is_immutable_array", - Value: builtinIsImmutableArray, - }, - { - Name: "is_map", - Value: builtinIsMap, - }, - { - Name: "is_immutable_map", - Value: builtinIsImmutableMap, - }, - { - Name: "is_iterable", - Value: builtinIsIterable, - }, - { - Name: "is_time", - Value: builtinIsTime, - }, - { - Name: "is_error", - Value: builtinIsError, - }, - { - Name: "is_undefined", - Value: builtinIsUndefined, - }, - { - Name: "is_function", - Value: builtinIsFunction, - }, - { - Name: "is_callable", - Value: builtinIsCallable, - }, - { - Name: "type_name", - Value: builtinTypeName, - }, - { - Name: "format", - Value: builtinFormat, - }, - { - Name: "range", - Value: builtinRange, - }, +var builtinFuncs []*BuiltinFunction + +func addBuiltinFunction(name string, fn CallableFunc) { + builtinFuncs = append(builtinFuncs, &BuiltinFunction{Name: name, Value: fn}) +} + +func init() { + addBuiltinFunction("len", builtinLen) + addBuiltinFunction("copy", builtinCopy) + addBuiltinFunction("append", builtinAppend) + addBuiltinFunction("delete", builtinDelete) + addBuiltinFunction("splice", builtinSplice) + addBuiltinFunction("string", builtinString) + addBuiltinFunction("int", builtinInt) + addBuiltinFunction("bool", builtinBool) + addBuiltinFunction("float", builtinFloat) + addBuiltinFunction("char", builtinChar) + addBuiltinFunction("bytes", builtinBytes) + addBuiltinFunction("time", builtinTime) + addBuiltinFunction("is_int", builtinIsInt) + addBuiltinFunction("is_float", builtinIsFloat) + addBuiltinFunction("is_string", builtinIsString) + addBuiltinFunction("is_bool", builtinIsBool) + addBuiltinFunction("is_char", builtinIsChar) + addBuiltinFunction("is_bytes", builtinIsBytes) + addBuiltinFunction("is_array", builtinIsArray) + addBuiltinFunction("is_immutable_array", builtinIsImmutableArray) + addBuiltinFunction("is_map", builtinIsMap) + addBuiltinFunction("is_immutable_map", builtinIsImmutableMap) + addBuiltinFunction("is_iterable", builtinIsIterable) + addBuiltinFunction("is_time", builtinIsTime) + addBuiltinFunction("is_error", builtinIsError) + addBuiltinFunction("is_undefined", builtinIsUndefined) + addBuiltinFunction("is_function", builtinIsFunction) + addBuiltinFunction("is_callable", builtinIsCallable) + addBuiltinFunction("type_name", builtinTypeName) + addBuiltinFunction("format", builtinFormat) + addBuiltinFunction("range", builtinRange) } // GetAllBuiltinFunctions returns all builtin function objects. @@ -133,6 +46,7 @@ func GetAllBuiltinFunctions() []*BuiltinFunction { } func builtinTypeName(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -140,6 +54,7 @@ func builtinTypeName(args ...Object) (Object, error) { } func builtinIsString(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -150,6 +65,7 @@ func builtinIsString(args ...Object) (Object, error) { } func builtinIsInt(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -160,6 +76,7 @@ func builtinIsInt(args ...Object) (Object, error) { } func builtinIsFloat(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -170,6 +87,7 @@ func builtinIsFloat(args ...Object) (Object, error) { } func builtinIsBool(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -180,6 +98,7 @@ func builtinIsBool(args ...Object) (Object, error) { } func builtinIsChar(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -190,6 +109,7 @@ func builtinIsChar(args ...Object) (Object, error) { } func builtinIsBytes(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -200,6 +120,7 @@ func builtinIsBytes(args ...Object) (Object, error) { } func builtinIsArray(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -210,6 +131,7 @@ func builtinIsArray(args ...Object) (Object, error) { } func builtinIsImmutableArray(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -220,6 +142,7 @@ func builtinIsImmutableArray(args ...Object) (Object, error) { } func builtinIsMap(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -230,6 +153,7 @@ func builtinIsMap(args ...Object) (Object, error) { } func builtinIsImmutableMap(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -240,6 +164,7 @@ func builtinIsImmutableMap(args ...Object) (Object, error) { } func builtinIsTime(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -250,6 +175,7 @@ func builtinIsTime(args ...Object) (Object, error) { } func builtinIsError(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -260,6 +186,7 @@ func builtinIsError(args ...Object) (Object, error) { } func builtinIsUndefined(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -270,6 +197,7 @@ func builtinIsUndefined(args ...Object) (Object, error) { } func builtinIsFunction(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -281,6 +209,7 @@ func builtinIsFunction(args ...Object) (Object, error) { } func builtinIsCallable(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -291,6 +220,7 @@ func builtinIsCallable(args ...Object) (Object, error) { } func builtinIsIterable(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -302,6 +232,7 @@ func builtinIsIterable(args ...Object) (Object, error) { // len(obj object) => int func builtinLen(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -329,6 +260,7 @@ func builtinLen(args ...Object) (Object, error) { //range(start, stop[, step]) func builtinRange(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM numArgs := len(args) if numArgs < 2 || numArgs > 3 { return nil, ErrWrongNumArguments @@ -393,6 +325,7 @@ func buildRange(start, stop, step int64) *Array { } func builtinFormat(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM numArgs := len(args) if numArgs == 0 { return nil, ErrWrongNumArguments @@ -417,6 +350,7 @@ func builtinFormat(args ...Object) (Object, error) { } func builtinCopy(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -424,6 +358,7 @@ func builtinCopy(args ...Object) (Object, error) { } func builtinString(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -445,6 +380,7 @@ func builtinString(args ...Object) (Object, error) { } func builtinInt(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -463,6 +399,7 @@ func builtinInt(args ...Object) (Object, error) { } func builtinFloat(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -481,6 +418,7 @@ func builtinFloat(args ...Object) (Object, error) { } func builtinBool(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -498,6 +436,7 @@ func builtinBool(args ...Object) (Object, error) { } func builtinChar(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -516,6 +455,7 @@ func builtinChar(args ...Object) (Object, error) { } func builtinBytes(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -542,6 +482,7 @@ func builtinBytes(args ...Object) (Object, error) { } func builtinTime(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -561,6 +502,7 @@ func builtinTime(args ...Object) (Object, error) { // append(arr, items...) func builtinAppend(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM if len(args) < 2 { return nil, ErrWrongNumArguments } @@ -582,6 +524,7 @@ func builtinAppend(args ...Object) (Object, error) { // usage: delete(map, "key") // key must be a string func builtinDelete(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if argsLen != 2 { return nil, ErrWrongNumArguments @@ -610,6 +553,7 @@ func builtinDelete(args ...Object) (Object, error) { // usage: // deleted_items := splice(array[,start[,delete_count[,item1[,item2[,...]]]]) func builtinSplice(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM argsLen := len(args) if argsLen == 0 { return nil, ErrWrongNumArguments diff --git a/jobchan.go b/jobchan.go new file mode 100644 index 00000000..6336de5c --- /dev/null +++ b/jobchan.go @@ -0,0 +1,211 @@ +package tengo + +import ( + "time" +) + +func init() { + addBuiltinFunction("go", builtinGo) + addBuiltinFunction("makechan", builtinMakechan) +} + +type result struct { + retVal Object + err error +} + +type job struct { + result + vm *VM + waitChan chan result + done bool +} + +// Start a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from the current running VM. +// Return a job object that has wait, result, abort methods. +func builtinGo(args ...Object) (Object, error) { + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) == 0 { + return nil, ErrWrongNumArguments + } + fn, ok := args[0].(*CompiledFunction) + if !ok { + return nil, ErrInvalidArgumentType{ + Name: "first", + Expected: "func", + Found: args[0].TypeName(), + } + } + + newVM := vm.ShallowClone() + jb := &job{ + vm: newVM, + waitChan: make(chan result), + } + + go func() { + retVal, err := jb.vm.RunCompiled(fn, args[1:]...) + jb.waitChan <- result{retVal, err} + }() + + obj := map[string]Object{ + "result": &BuiltinFunction{Value: jb.getResult}, + "wait": &BuiltinFunction{Value: jb.waitTimeout}, + "abort": &BuiltinFunction{Value: jb.abort}, + } + return &Map{Value: obj}, nil +} + +// Return true if job is done +func (jb *job) wait(seconds int64) bool { + if jb.done { + return true + } + + if seconds <= 0 { + seconds = 3153600000 // 100 years + } + + select { + case jb.result = <-jb.waitChan: + jb.done = true + case <-time.After(time.Duration(seconds) * time.Second): + return false + } + + return true +} + +// Wait the job to complete. +// Wait can have optional timeout in seconds if the first arg is int. +// Wait forever if the optional timeout not specified, or timeout <= 0 +// Return true if the job exited(successfully or not) within the timeout peroid. +func (jb *job) waitTimeout(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) > 1 { + return nil, ErrWrongNumArguments + } + timeOut := -1 + if len(args) == 1 { + t, ok := ToInt(args[0]) + if !ok { + return nil, ErrInvalidArgumentType{ + Name: "first", + Expected: "int(compatible)", + Found: args[0].TypeName(), + } + } + timeOut = t + } + + if jb.wait(int64(timeOut)) { + return TrueValue, nil + } + return FalseValue, nil +} + +func (jb *job) abort(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) != 0 { + return nil, ErrWrongNumArguments + } + + jb.vm.Abort() + return nil, nil +} + +// Wait job to complete, return Error value when jb.err is present, +// otherwise return the result value of fn(arg1, arg2, ...) +func (jb *job) getResult(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) != 0 { + return nil, ErrWrongNumArguments + } + + jb.wait(-1) + if jb.err != nil { + return &Error{Value: &String{Value: jb.err.Error()}}, nil + } + + return jb.retVal, nil +} + +type objchan chan Object + +// Make a channel to send/receive object +// Return a chan object that has send, recv, close methods. +func builtinMakechan(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM + var size int + switch len(args) { + case 0: + case 1: + n, ok := ToInt(args[0]) + if !ok { + return nil, ErrInvalidArgumentType{ + Name: "first", + Expected: "int(compatible)", + Found: args[0].TypeName(), + } + } + size = n + default: + return nil, ErrWrongNumArguments + } + + oc := make(objchan, size) + obj := map[string]Object{ + "send": &BuiltinFunction{Value: oc.send}, + "recv": &BuiltinFunction{Value: oc.recv}, + "close": &BuiltinFunction{Value: oc.close}, + } + return &Map{Value: obj}, nil +} + +// Send an obj to the channel, will block until channel is not full or (*VM).Abort() has been called. +// Send to a closed channel causes panic. +func (oc objchan) send(args ...Object) (Object, error) { + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) != 1 { + return nil, ErrWrongNumArguments + } + select { + case <-vm.ctx.Done(): + return nil, vm.ctx.Err() + //return &String{Value: vm.ctx.Err().Error()}, nil + case oc <- args[0]: + } + return nil, nil +} + +// Receive an obj from the channel, will block until channel is not empty or (*VM).Abort() has been called. +// Receive from a closed channel returns undefined value. +func (oc objchan) recv(args ...Object) (Object, error) { + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) != 0 { + return nil, ErrWrongNumArguments + } + select { + case <-vm.ctx.Done(): + return nil, vm.ctx.Err() + //return &String{Value: vm.ctx.Err().Error()}, nil + case obj, ok := <-oc: + if ok { + return obj, nil + } + } + return nil, nil +} + +// Close the channel. +func (oc objchan) close(args ...Object) (Object, error) { + args = args[1:] // the first arg is VMObj inserted by VM + if len(args) != 0 { + return nil, ErrWrongNumArguments + } + close(oc) + return nil, nil +} diff --git a/vm.go b/vm.go index 811ecef9..60af4772 100644 --- a/vm.go +++ b/vm.go @@ -1,6 +1,7 @@ package tengo import ( + "context" "fmt" "sync/atomic" @@ -16,8 +17,14 @@ type frame struct { basePointer int } +type cancelCtx struct { + context.Context + cancel context.CancelFunc +} + // VM is a virtual machine that executes the bytecode compiled by Compiler. type VM struct { + ctx cancelCtx // used to signal blocked channel sending and receiving to exit constants []Object stack [StackSize]Object sp int @@ -52,6 +59,8 @@ func NewVM( ip: -1, maxAllocs: maxAllocs, } + ctx, cancel := context.WithCancel(context.Background()) + v.ctx = cancelCtx{ctx, cancel} v.frames[0].fn = bytecode.MainFunction v.frames[0].ip = -1 v.curFrame = &v.frames[0] @@ -62,10 +71,11 @@ func NewVM( // Abort aborts the execution. func (v *VM) Abort() { atomic.StoreInt64(&v.aborting, 1) + v.ctx.cancel() } // Run starts the execution. -func (v *VM) Run() (err error) { +func (v *VM) Run() error { // reset VM states v.sp = 0 v.curFrame = &(v.frames[0]) @@ -75,7 +85,10 @@ func (v *VM) Run() (err error) { v.allocs = v.maxAllocs + 1 v.run() - atomic.StoreInt64(&v.aborting, 0) + return v.postRun() +} + +func (v *VM) postRun() (err error) { err = v.err if err != nil { filePos := v.fileSet.Position( @@ -89,9 +102,108 @@ func (v *VM) Run() (err error) { v.curFrame.fn.SourcePos(v.curFrame.ip - 1)) err = fmt.Errorf("%w\n\tat %s", err, filePos) } - return err } - return nil + + if atomic.LoadInt64(&v.aborting) == 1 { + if err != nil { + err = fmt.Errorf("VM aborted\n\t%w", err) + } else { + err = fmt.Errorf("VM aborted") + } + } + atomic.StoreInt64(&v.aborting, 0) + return err +} + +func concatInsts(instructions ...[]byte) []byte { + var concat []byte + for _, i := range instructions { + concat = append(concat, i...) + } + return concat +} + +var emptyEntry = &CompiledFunction{ + Instructions: MakeInstruction(parser.OpSuspend), +} + +// ShallowClone creates a shallow copy of the current VM, with separate stack and frame. +// The copy shares the underlying globals, constants with the original. +// ShallowClone is typically followed by RunCompiled to run user supplied compiled function. +func (v *VM) ShallowClone() *VM { + shallowClone := &VM{ + constants: v.constants, + sp: 0, + globals: v.globals, + fileSet: v.fileSet, + framesIndex: 1, + ip: -1, + maxAllocs: v.maxAllocs, + } + + //ctx, cancel := context.WithCancel(v.ctx.Context) // cloned VM derives the context of its parent + ctx, cancel := context.WithCancel(context.Background()) // cloned VM has independent context + shallowClone.ctx = cancelCtx{ctx, cancel} + // set to empty entry + shallowClone.frames[0].fn = emptyEntry + shallowClone.frames[0].ip = -1 + shallowClone.curFrame = &v.frames[0] + shallowClone.curInsts = v.curFrame.fn.Instructions + + return shallowClone +} + +// constract wrapper function func(fn, ...args){ return fn(args...) } +var funcWrapper = &CompiledFunction{ + Instructions: concatInsts( + MakeInstruction(parser.OpGetLocal, 0), + MakeInstruction(parser.OpGetLocal, 1), + MakeInstruction(parser.OpCall, 1, 1), + MakeInstruction(parser.OpReturn, 1), + ), + NumLocals: 2, + NumParameters: 2, + VarArgs: true, +} + +// RunCompiled run the VM with user supplied function fn. +func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (Object, error) { + entry := &CompiledFunction{ + Instructions: concatInsts( + MakeInstruction(parser.OpCall, 1+len(args), 0), + MakeInstruction(parser.OpSuspend), + ), + } + + v.stack[0] = funcWrapper + v.stack[1] = fn + for i, arg := range args { + v.stack[i+2] = arg + } + v.sp = 2 + len(args) + + v.frames[0].fn = entry + v.frames[0].ip = -1 + v.curFrame = &v.frames[0] + v.curInsts = v.curFrame.fn.Instructions + v.framesIndex = 1 + v.ip = -1 + v.allocs = v.maxAllocs + 1 + + v.run() + if err := v.postRun(); err != nil { + return UndefinedValue, err + } + return v.stack[v.sp-1], nil +} + +type VMObj struct { + ObjectImpl + Value *VM +} + +func (v *VM) selfObject() Object { + return &VMObj{Value: v} } func (v *VM) run() { @@ -629,6 +741,10 @@ func (v *VM) run() { v.sp = v.sp - numArgs + callee.NumLocals } else { var args []Object + if _, ok := value.(*BuiltinFunction); ok { + // pass VM as the first para to builtin functions + args = append(args, v.selfObject()) + } args = append(args, v.stack[v.sp-numArgs:v.sp]...) ret, e := value.Call(args...) v.sp -= numArgs + 1 From 67ce5952e85a6fa36622fe6977917530acb8f0c5 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Sat, 27 Mar 2021 18:41:15 +0000 Subject: [PATCH 02/20] Rename job to goroutineVM for better understanding A goroutineVM is a VM cloned from the current VM in a shared way that the cloned VM(the child) can access globals and constants of the parent VM, but with its own stack and frames. The cloned VM will be running with a specified compiled function as the entrypoint in a new goroutine created by the parent VM. --- jobchan.go => goroutinevm.go | 72 ++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 35 deletions(-) rename jobchan.go => goroutinevm.go (72%) diff --git a/jobchan.go b/goroutinevm.go similarity index 72% rename from jobchan.go rename to goroutinevm.go index 6336de5c..fe28c327 100644 --- a/jobchan.go +++ b/goroutinevm.go @@ -1,6 +1,7 @@ package tengo import ( + "sync/atomic" "time" ) @@ -9,20 +10,20 @@ func init() { addBuiltinFunction("makechan", builtinMakechan) } -type result struct { - retVal Object - err error +type ret struct { + val Object + err error } -type job struct { - result - vm *VM - waitChan chan result - done bool +type goroutineVM struct { + *VM + ret // return value of (*VM).RunCompiled() + waitChan chan ret + done int64 } // Start a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from the current running VM. -// Return a job object that has wait, result, abort methods. +// Return a goroutineVM object that has wait, result, abort methods. func builtinGo(args ...Object) (Object, error) { vm := args[0].(*VMObj).Value args = args[1:] // the first arg is VMObj inserted by VM @@ -39,27 +40,27 @@ func builtinGo(args ...Object) (Object, error) { } newVM := vm.ShallowClone() - jb := &job{ - vm: newVM, - waitChan: make(chan result), + gvm := &goroutineVM{ + VM: newVM, + waitChan: make(chan ret), } go func() { - retVal, err := jb.vm.RunCompiled(fn, args[1:]...) - jb.waitChan <- result{retVal, err} + val, err := gvm.RunCompiled(fn, args[1:]...) + gvm.waitChan <- ret{val, err} }() obj := map[string]Object{ - "result": &BuiltinFunction{Value: jb.getResult}, - "wait": &BuiltinFunction{Value: jb.waitTimeout}, - "abort": &BuiltinFunction{Value: jb.abort}, + "result": &BuiltinFunction{Value: gvm.getRet}, + "wait": &BuiltinFunction{Value: gvm.waitTimeout}, + "abort": &BuiltinFunction{Value: gvm.abort}, } return &Map{Value: obj}, nil } -// Return true if job is done -func (jb *job) wait(seconds int64) bool { - if jb.done { +// Return true if the goroutineVM is done +func (gvm *goroutineVM) wait(seconds int64) bool { + if atomic.LoadInt64(&gvm.done) == 1 { return true } @@ -68,8 +69,8 @@ func (jb *job) wait(seconds int64) bool { } select { - case jb.result = <-jb.waitChan: - jb.done = true + case gvm.ret = <-gvm.waitChan: + atomic.StoreInt64(&gvm.done, 1) case <-time.After(time.Duration(seconds) * time.Second): return false } @@ -77,11 +78,11 @@ func (jb *job) wait(seconds int64) bool { return true } -// Wait the job to complete. +// Wait for the goroutineVM to complete. // Wait can have optional timeout in seconds if the first arg is int. // Wait forever if the optional timeout not specified, or timeout <= 0 -// Return true if the job exited(successfully or not) within the timeout peroid. -func (jb *job) waitTimeout(args ...Object) (Object, error) { +// Return true if the goroutineVM exited(successfully or not) within the timeout peroid. +func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { args = args[1:] // the first arg is VMObj inserted by VM if len(args) > 1 { return nil, ErrWrongNumArguments @@ -99,36 +100,37 @@ func (jb *job) waitTimeout(args ...Object) (Object, error) { timeOut = t } - if jb.wait(int64(timeOut)) { + if gvm.wait(int64(timeOut)) { return TrueValue, nil } return FalseValue, nil } -func (jb *job) abort(args ...Object) (Object, error) { +// Terminate the execution of the goroutineVM. +func (gvm *goroutineVM) abort(args ...Object) (Object, error) { args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } - jb.vm.Abort() + gvm.Abort() return nil, nil } -// Wait job to complete, return Error value when jb.err is present, -// otherwise return the result value of fn(arg1, arg2, ...) -func (jb *job) getResult(args ...Object) (Object, error) { +// Wait the goroutineVM to complete, return Error object if any runtime error occurred +// during the execution, otherwise return the result value of fn(arg1, arg2, ...) +func (gvm *goroutineVM) getRet(args ...Object) (Object, error) { args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } - jb.wait(-1) - if jb.err != nil { - return &Error{Value: &String{Value: jb.err.Error()}}, nil + gvm.wait(-1) + if gvm.ret.err != nil { + return &Error{Value: &String{Value: gvm.ret.err.Error()}}, nil } - return jb.retVal, nil + return gvm.ret.val, nil } type objchan chan Object From d67644ec987acfa0547ade80f9f4a486b2a794b7 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Mon, 29 Mar 2021 15:24:26 +0000 Subject: [PATCH 03/20] Rename builtin go to govm In golang spec, "go" statement has no return value, which is not the semantic here: we want "govm" returns a goroutineVM object which can be used to wait, abort, or get result of the goroutineVM. --- goroutinevm.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/goroutinevm.go b/goroutinevm.go index fe28c327..3903a304 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -6,7 +6,7 @@ import ( ) func init() { - addBuiltinFunction("go", builtinGo) + addBuiltinFunction("govm", builtinGovm) addBuiltinFunction("makechan", builtinMakechan) } @@ -24,7 +24,7 @@ type goroutineVM struct { // Start a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from the current running VM. // Return a goroutineVM object that has wait, result, abort methods. -func builtinGo(args ...Object) (Object, error) { +func builtinGovm(args ...Object) (Object, error) { vm := args[0].(*VMObj).Value args = args[1:] // the first arg is VMObj inserted by VM if len(args) == 0 { @@ -64,7 +64,7 @@ func (gvm *goroutineVM) wait(seconds int64) bool { return true } - if seconds <= 0 { + if seconds < 0 { seconds = 3153600000 // 100 years } @@ -78,10 +78,9 @@ func (gvm *goroutineVM) wait(seconds int64) bool { return true } -// Wait for the goroutineVM to complete. -// Wait can have optional timeout in seconds if the first arg is int. -// Wait forever if the optional timeout not specified, or timeout <= 0 +// Wait for the goroutineVM to complete in timeout seconds. // Return true if the goroutineVM exited(successfully or not) within the timeout peroid. +// Wait forever if the optional timeout not specified, or timeout < 0. func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { args = args[1:] // the first arg is VMObj inserted by VM if len(args) > 1 { From e4ca97e935d67011f865637c8c03da239825dd47 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Mon, 29 Mar 2021 16:10:16 +0000 Subject: [PATCH 04/20] Add docs for builtin "govm" and "makechan" fix typo --- docs/builtins.md | 104 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/docs/builtins.md b/docs/builtins.md index 940a215a..e9044544 100644 --- a/docs/builtins.md +++ b/docs/builtins.md @@ -126,6 +126,110 @@ v := ["a", "b", "c"] items := splice(v, 1, 1, "d", "e") // items == ["b"], v == ["a", "d", "e", "c"] ``` +## govm + +Starts a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from +the current running VM, and returns a goroutineVM object that has +wait, result, abort methods. + +```golang +var := 0 + +f1 := func(a,b) { var = 10; return a+b } +f2 := func(a,b,c) { var = 11; return a+b+c } + +gvm1 := govm(f1,1,2) +gvm2 := govm(f2,1,2,5) + +fmt.println(gvm1.result()) // 3 +fmt.println(gvm2.result()) // 8 +fmt.println(var) // 10 or 11 +``` + +Below is a simple client server example: + +```golang +reqChan := makechan(8) +repChan := makechan(8) + +client := func(interval) { + reqChan.send("hello") + i := 0 + for { + fmt.println(repChan.recv()) + times.sleep(interval*times.second) + reqChan.send(i) + i++ + } +} + +server := func() { + for { + req := reqChan.recv() + if req == "hello" { + fmt.println(req) + repChan.send("world") + } else { + repChan.send(req+100) + } + } +} + +gvmClient := govm(client, 2) +gvmServer := govm(server) + +if ok := gvmClient.wait(5); !ok { + gvmClient.abort() +} +gvmServer.abort() + +fmt.println("client: ", gvmClient.result()) +fmt.println("server: ", gvmServer.result()) + +//output: +//hello +//world +//100 +//101 +//client: error: "VM aborted" +//server: error: "VM aborted\n\tRuntime Error: context canceled at -\n\tat -" +``` + +* wait() waits for the goroutineVM to complete in timeout seconds and +returns true if the goroutineVM exited(successfully or not) within the +timeout peroid. It waits forever if the optional timeout not specified, +or timeout < 0. +* abort() terminates the VM. +* result() waits the goroutineVM to complete, returns Error object if +any runtime error occurred during the execution, otherwise returns the +result value of fn(arg1, arg2, ...) + +## makechan + +Makes a channel to send/receive object and returns a chan object that has +send, recv, close methods. + +```golang +unbufferedChan := makechan() +bufferedChan := makechan(128) + +// Send will block if the channel is full. +bufferedChan.send("hello") // send string +bufferedChan.send(55) // send int +bufferedChan.send([66, makechan(1)]) // channel in channel + +// Receive will block if the channel is empty. +obj := bufferedChan.recv() + +// Send to a closed channel causes panic. +// Receive from a closed channel returns undefined value. +unbufferedChan.close() +bufferedChan.close() +``` + +On the time the VM that the chan is running in is aborted, the sending +or receiving call returns immediately. + ## type_name Returns the type_name of an object. From ee0878271223457f2ffd7be2d9661e45553e40a9 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Mon, 29 Mar 2021 16:57:44 +0000 Subject: [PATCH 05/20] Use unexported vmObj --- builtins.go | 62 +++++++++++++++++++++++++------------------------- goroutinevm.go | 22 +++++++++--------- vm.go | 4 ++-- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/builtins.go b/builtins.go index 2e247390..e17ffb96 100644 --- a/builtins.go +++ b/builtins.go @@ -46,7 +46,7 @@ func GetAllBuiltinFunctions() []*BuiltinFunction { } func builtinTypeName(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -54,7 +54,7 @@ func builtinTypeName(args ...Object) (Object, error) { } func builtinIsString(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -65,7 +65,7 @@ func builtinIsString(args ...Object) (Object, error) { } func builtinIsInt(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -76,7 +76,7 @@ func builtinIsInt(args ...Object) (Object, error) { } func builtinIsFloat(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -87,7 +87,7 @@ func builtinIsFloat(args ...Object) (Object, error) { } func builtinIsBool(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -98,7 +98,7 @@ func builtinIsBool(args ...Object) (Object, error) { } func builtinIsChar(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -109,7 +109,7 @@ func builtinIsChar(args ...Object) (Object, error) { } func builtinIsBytes(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -120,7 +120,7 @@ func builtinIsBytes(args ...Object) (Object, error) { } func builtinIsArray(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -131,7 +131,7 @@ func builtinIsArray(args ...Object) (Object, error) { } func builtinIsImmutableArray(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -142,7 +142,7 @@ func builtinIsImmutableArray(args ...Object) (Object, error) { } func builtinIsMap(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -153,7 +153,7 @@ func builtinIsMap(args ...Object) (Object, error) { } func builtinIsImmutableMap(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -164,7 +164,7 @@ func builtinIsImmutableMap(args ...Object) (Object, error) { } func builtinIsTime(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -175,7 +175,7 @@ func builtinIsTime(args ...Object) (Object, error) { } func builtinIsError(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -186,7 +186,7 @@ func builtinIsError(args ...Object) (Object, error) { } func builtinIsUndefined(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -197,7 +197,7 @@ func builtinIsUndefined(args ...Object) (Object, error) { } func builtinIsFunction(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -209,7 +209,7 @@ func builtinIsFunction(args ...Object) (Object, error) { } func builtinIsCallable(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -220,7 +220,7 @@ func builtinIsCallable(args ...Object) (Object, error) { } func builtinIsIterable(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -232,7 +232,7 @@ func builtinIsIterable(args ...Object) (Object, error) { // len(obj object) => int func builtinLen(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -260,7 +260,7 @@ func builtinLen(args ...Object) (Object, error) { //range(start, stop[, step]) func builtinRange(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM numArgs := len(args) if numArgs < 2 || numArgs > 3 { return nil, ErrWrongNumArguments @@ -325,7 +325,7 @@ func buildRange(start, stop, step int64) *Array { } func builtinFormat(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM numArgs := len(args) if numArgs == 0 { return nil, ErrWrongNumArguments @@ -350,7 +350,7 @@ func builtinFormat(args ...Object) (Object, error) { } func builtinCopy(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -358,7 +358,7 @@ func builtinCopy(args ...Object) (Object, error) { } func builtinString(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -380,7 +380,7 @@ func builtinString(args ...Object) (Object, error) { } func builtinInt(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -399,7 +399,7 @@ func builtinInt(args ...Object) (Object, error) { } func builtinFloat(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -418,7 +418,7 @@ func builtinFloat(args ...Object) (Object, error) { } func builtinBool(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -436,7 +436,7 @@ func builtinBool(args ...Object) (Object, error) { } func builtinChar(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -455,7 +455,7 @@ func builtinChar(args ...Object) (Object, error) { } func builtinBytes(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -482,7 +482,7 @@ func builtinBytes(args ...Object) (Object, error) { } func builtinTime(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -502,7 +502,7 @@ func builtinTime(args ...Object) (Object, error) { // append(arr, items...) func builtinAppend(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) < 2 { return nil, ErrWrongNumArguments } @@ -524,7 +524,7 @@ func builtinAppend(args ...Object) (Object, error) { // usage: delete(map, "key") // key must be a string func builtinDelete(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if argsLen != 2 { return nil, ErrWrongNumArguments @@ -553,7 +553,7 @@ func builtinDelete(args ...Object) (Object, error) { // usage: // deleted_items := splice(array[,start[,delete_count[,item1[,item2[,...]]]]) func builtinSplice(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if argsLen == 0 { return nil, ErrWrongNumArguments diff --git a/goroutinevm.go b/goroutinevm.go index 3903a304..a5236454 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -25,8 +25,8 @@ type goroutineVM struct { // Start a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from the current running VM. // Return a goroutineVM object that has wait, result, abort methods. func builtinGovm(args ...Object) (Object, error) { - vm := args[0].(*VMObj).Value - args = args[1:] // the first arg is VMObj inserted by VM + vm := args[0].(*vmObj).Value + args = args[1:] // the first arg is vmObj inserted by VM if len(args) == 0 { return nil, ErrWrongNumArguments } @@ -82,7 +82,7 @@ func (gvm *goroutineVM) wait(seconds int64) bool { // Return true if the goroutineVM exited(successfully or not) within the timeout peroid. // Wait forever if the optional timeout not specified, or timeout < 0. func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) > 1 { return nil, ErrWrongNumArguments } @@ -107,7 +107,7 @@ func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { // Terminate the execution of the goroutineVM. func (gvm *goroutineVM) abort(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } @@ -119,7 +119,7 @@ func (gvm *goroutineVM) abort(args ...Object) (Object, error) { // Wait the goroutineVM to complete, return Error object if any runtime error occurred // during the execution, otherwise return the result value of fn(arg1, arg2, ...) func (gvm *goroutineVM) getRet(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } @@ -137,7 +137,7 @@ type objchan chan Object // Make a channel to send/receive object // Return a chan object that has send, recv, close methods. func builtinMakechan(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM var size int switch len(args) { case 0: @@ -167,8 +167,8 @@ func builtinMakechan(args ...Object) (Object, error) { // Send an obj to the channel, will block until channel is not full or (*VM).Abort() has been called. // Send to a closed channel causes panic. func (oc objchan) send(args ...Object) (Object, error) { - vm := args[0].(*VMObj).Value - args = args[1:] // the first arg is VMObj inserted by VM + vm := args[0].(*vmObj).Value + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -184,8 +184,8 @@ func (oc objchan) send(args ...Object) (Object, error) { // Receive an obj from the channel, will block until channel is not empty or (*VM).Abort() has been called. // Receive from a closed channel returns undefined value. func (oc objchan) recv(args ...Object) (Object, error) { - vm := args[0].(*VMObj).Value - args = args[1:] // the first arg is VMObj inserted by VM + vm := args[0].(*vmObj).Value + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } @@ -203,7 +203,7 @@ func (oc objchan) recv(args ...Object) (Object, error) { // Close the channel. func (oc objchan) close(args ...Object) (Object, error) { - args = args[1:] // the first arg is VMObj inserted by VM + args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } diff --git a/vm.go b/vm.go index 60af4772..e86d2782 100644 --- a/vm.go +++ b/vm.go @@ -197,13 +197,13 @@ func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (Object, error) { return v.stack[v.sp-1], nil } -type VMObj struct { +type vmObj struct { ObjectImpl Value *VM } func (v *VM) selfObject() Object { - return &VMObj{Value: v} + return &vmObj{Value: v} } func (v *VM) run() { From 9e5fbbc9815c3fe32ff4a642ec01af9ec148a42d Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Tue, 30 Mar 2021 16:34:16 +0000 Subject: [PATCH 06/20] Use needvmObj flag in BuiltinFunction VM will only pass its vmObj to builtin function that has needvmObj equals true, so that normal builtin functions do not need to notice the change. --- builtins.go | 98 ++++++++++++++++++-------------------------------- goroutinevm.go | 13 +++---- objects.go | 5 +-- vm.go | 8 +++-- 4 files changed, 46 insertions(+), 78 deletions(-) diff --git a/builtins.go b/builtins.go index e17ffb96..b90da6c4 100644 --- a/builtins.go +++ b/builtins.go @@ -2,42 +2,43 @@ package tengo var builtinFuncs []*BuiltinFunction -func addBuiltinFunction(name string, fn CallableFunc) { - builtinFuncs = append(builtinFuncs, &BuiltinFunction{Name: name, Value: fn}) +// if needvmObj is true, VM will pass [vmObj, args...] to fn when calling it. +func addBuiltinFunction(name string, fn CallableFunc, needvmObj bool) { + builtinFuncs = append(builtinFuncs, &BuiltinFunction{Name: name, Value: fn, needvmObj: needvmObj}) } func init() { - addBuiltinFunction("len", builtinLen) - addBuiltinFunction("copy", builtinCopy) - addBuiltinFunction("append", builtinAppend) - addBuiltinFunction("delete", builtinDelete) - addBuiltinFunction("splice", builtinSplice) - addBuiltinFunction("string", builtinString) - addBuiltinFunction("int", builtinInt) - addBuiltinFunction("bool", builtinBool) - addBuiltinFunction("float", builtinFloat) - addBuiltinFunction("char", builtinChar) - addBuiltinFunction("bytes", builtinBytes) - addBuiltinFunction("time", builtinTime) - addBuiltinFunction("is_int", builtinIsInt) - addBuiltinFunction("is_float", builtinIsFloat) - addBuiltinFunction("is_string", builtinIsString) - addBuiltinFunction("is_bool", builtinIsBool) - addBuiltinFunction("is_char", builtinIsChar) - addBuiltinFunction("is_bytes", builtinIsBytes) - addBuiltinFunction("is_array", builtinIsArray) - addBuiltinFunction("is_immutable_array", builtinIsImmutableArray) - addBuiltinFunction("is_map", builtinIsMap) - addBuiltinFunction("is_immutable_map", builtinIsImmutableMap) - addBuiltinFunction("is_iterable", builtinIsIterable) - addBuiltinFunction("is_time", builtinIsTime) - addBuiltinFunction("is_error", builtinIsError) - addBuiltinFunction("is_undefined", builtinIsUndefined) - addBuiltinFunction("is_function", builtinIsFunction) - addBuiltinFunction("is_callable", builtinIsCallable) - addBuiltinFunction("type_name", builtinTypeName) - addBuiltinFunction("format", builtinFormat) - addBuiltinFunction("range", builtinRange) + addBuiltinFunction("len", builtinLen, false) + addBuiltinFunction("copy", builtinCopy, false) + addBuiltinFunction("append", builtinAppend, false) + addBuiltinFunction("delete", builtinDelete, false) + addBuiltinFunction("splice", builtinSplice, false) + addBuiltinFunction("string", builtinString, false) + addBuiltinFunction("int", builtinInt, false) + addBuiltinFunction("bool", builtinBool, false) + addBuiltinFunction("float", builtinFloat, false) + addBuiltinFunction("char", builtinChar, false) + addBuiltinFunction("bytes", builtinBytes, false) + addBuiltinFunction("time", builtinTime, false) + addBuiltinFunction("is_int", builtinIsInt, false) + addBuiltinFunction("is_float", builtinIsFloat, false) + addBuiltinFunction("is_string", builtinIsString, false) + addBuiltinFunction("is_bool", builtinIsBool, false) + addBuiltinFunction("is_char", builtinIsChar, false) + addBuiltinFunction("is_bytes", builtinIsBytes, false) + addBuiltinFunction("is_array", builtinIsArray, false) + addBuiltinFunction("is_immutable_array", builtinIsImmutableArray, false) + addBuiltinFunction("is_map", builtinIsMap, false) + addBuiltinFunction("is_immutable_map", builtinIsImmutableMap, false) + addBuiltinFunction("is_iterable", builtinIsIterable, false) + addBuiltinFunction("is_time", builtinIsTime, false) + addBuiltinFunction("is_error", builtinIsError, false) + addBuiltinFunction("is_undefined", builtinIsUndefined, false) + addBuiltinFunction("is_function", builtinIsFunction, false) + addBuiltinFunction("is_callable", builtinIsCallable, false) + addBuiltinFunction("type_name", builtinTypeName, false) + addBuiltinFunction("format", builtinFormat, false) + addBuiltinFunction("range", builtinRange, false) } // GetAllBuiltinFunctions returns all builtin function objects. @@ -46,7 +47,6 @@ func GetAllBuiltinFunctions() []*BuiltinFunction { } func builtinTypeName(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -54,7 +54,6 @@ func builtinTypeName(args ...Object) (Object, error) { } func builtinIsString(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -65,7 +64,6 @@ func builtinIsString(args ...Object) (Object, error) { } func builtinIsInt(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -76,7 +74,6 @@ func builtinIsInt(args ...Object) (Object, error) { } func builtinIsFloat(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -87,7 +84,6 @@ func builtinIsFloat(args ...Object) (Object, error) { } func builtinIsBool(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -98,7 +94,6 @@ func builtinIsBool(args ...Object) (Object, error) { } func builtinIsChar(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -109,7 +104,6 @@ func builtinIsChar(args ...Object) (Object, error) { } func builtinIsBytes(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -120,7 +114,6 @@ func builtinIsBytes(args ...Object) (Object, error) { } func builtinIsArray(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -131,7 +124,6 @@ func builtinIsArray(args ...Object) (Object, error) { } func builtinIsImmutableArray(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -142,7 +134,6 @@ func builtinIsImmutableArray(args ...Object) (Object, error) { } func builtinIsMap(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -153,7 +144,6 @@ func builtinIsMap(args ...Object) (Object, error) { } func builtinIsImmutableMap(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -164,7 +154,6 @@ func builtinIsImmutableMap(args ...Object) (Object, error) { } func builtinIsTime(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -175,7 +164,6 @@ func builtinIsTime(args ...Object) (Object, error) { } func builtinIsError(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -186,7 +174,6 @@ func builtinIsError(args ...Object) (Object, error) { } func builtinIsUndefined(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -197,7 +184,6 @@ func builtinIsUndefined(args ...Object) (Object, error) { } func builtinIsFunction(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -209,7 +195,6 @@ func builtinIsFunction(args ...Object) (Object, error) { } func builtinIsCallable(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -220,7 +205,6 @@ func builtinIsCallable(args ...Object) (Object, error) { } func builtinIsIterable(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -232,7 +216,6 @@ func builtinIsIterable(args ...Object) (Object, error) { // len(obj object) => int func builtinLen(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -260,7 +243,6 @@ func builtinLen(args ...Object) (Object, error) { //range(start, stop[, step]) func builtinRange(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM numArgs := len(args) if numArgs < 2 || numArgs > 3 { return nil, ErrWrongNumArguments @@ -325,7 +307,6 @@ func buildRange(start, stop, step int64) *Array { } func builtinFormat(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM numArgs := len(args) if numArgs == 0 { return nil, ErrWrongNumArguments @@ -350,7 +331,6 @@ func builtinFormat(args ...Object) (Object, error) { } func builtinCopy(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -358,7 +338,6 @@ func builtinCopy(args ...Object) (Object, error) { } func builtinString(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -380,7 +359,6 @@ func builtinString(args ...Object) (Object, error) { } func builtinInt(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -399,7 +377,6 @@ func builtinInt(args ...Object) (Object, error) { } func builtinFloat(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -418,7 +395,6 @@ func builtinFloat(args ...Object) (Object, error) { } func builtinBool(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -436,7 +412,6 @@ func builtinBool(args ...Object) (Object, error) { } func builtinChar(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -455,7 +430,6 @@ func builtinChar(args ...Object) (Object, error) { } func builtinBytes(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -482,7 +456,6 @@ func builtinBytes(args ...Object) (Object, error) { } func builtinTime(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if !(argsLen == 1 || argsLen == 2) { return nil, ErrWrongNumArguments @@ -502,7 +475,6 @@ func builtinTime(args ...Object) (Object, error) { // append(arr, items...) func builtinAppend(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) < 2 { return nil, ErrWrongNumArguments } @@ -524,7 +496,6 @@ func builtinAppend(args ...Object) (Object, error) { // usage: delete(map, "key") // key must be a string func builtinDelete(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if argsLen != 2 { return nil, ErrWrongNumArguments @@ -553,7 +524,6 @@ func builtinDelete(args ...Object) (Object, error) { // usage: // deleted_items := splice(array[,start[,delete_count[,item1[,item2[,...]]]]) func builtinSplice(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM argsLen := len(args) if argsLen == 0 { return nil, ErrWrongNumArguments diff --git a/goroutinevm.go b/goroutinevm.go index a5236454..69b8c876 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -6,8 +6,8 @@ import ( ) func init() { - addBuiltinFunction("govm", builtinGovm) - addBuiltinFunction("makechan", builtinMakechan) + addBuiltinFunction("govm", builtinGovm, true) + addBuiltinFunction("makechan", builtinMakechan, false) } type ret struct { @@ -82,7 +82,6 @@ func (gvm *goroutineVM) wait(seconds int64) bool { // Return true if the goroutineVM exited(successfully or not) within the timeout peroid. // Wait forever if the optional timeout not specified, or timeout < 0. func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) > 1 { return nil, ErrWrongNumArguments } @@ -107,7 +106,6 @@ func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { // Terminate the execution of the goroutineVM. func (gvm *goroutineVM) abort(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } @@ -119,7 +117,6 @@ func (gvm *goroutineVM) abort(args ...Object) (Object, error) { // Wait the goroutineVM to complete, return Error object if any runtime error occurred // during the execution, otherwise return the result value of fn(arg1, arg2, ...) func (gvm *goroutineVM) getRet(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } @@ -137,7 +134,6 @@ type objchan chan Object // Make a channel to send/receive object // Return a chan object that has send, recv, close methods. func builtinMakechan(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM var size int switch len(args) { case 0: @@ -157,8 +153,8 @@ func builtinMakechan(args ...Object) (Object, error) { oc := make(objchan, size) obj := map[string]Object{ - "send": &BuiltinFunction{Value: oc.send}, - "recv": &BuiltinFunction{Value: oc.recv}, + "send": &BuiltinFunction{Value: oc.send, needvmObj: true}, + "recv": &BuiltinFunction{Value: oc.recv, needvmObj: true}, "close": &BuiltinFunction{Value: oc.close}, } return &Map{Value: obj}, nil @@ -203,7 +199,6 @@ func (oc objchan) recv(args ...Object) (Object, error) { // Close the channel. func (oc objchan) close(args ...Object) (Object, error) { - args = args[1:] // the first arg is vmObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } diff --git a/objects.go b/objects.go index 30913db5..416dcfc7 100644 --- a/objects.go +++ b/objects.go @@ -320,8 +320,9 @@ func (o *Bool) GobEncode() (b []byte, err error) { // BuiltinFunction represents a builtin function. type BuiltinFunction struct { ObjectImpl - Name string - Value CallableFunc + Name string + Value CallableFunc + needvmObj bool } // TypeName returns the name of the type. diff --git a/vm.go b/vm.go index e86d2782..3a4fd195 100644 --- a/vm.go +++ b/vm.go @@ -741,9 +741,11 @@ func (v *VM) run() { v.sp = v.sp - numArgs + callee.NumLocals } else { var args []Object - if _, ok := value.(*BuiltinFunction); ok { - // pass VM as the first para to builtin functions - args = append(args, v.selfObject()) + if bltnfn, ok := value.(*BuiltinFunction); ok { + if bltnfn.needvmObj { + // pass VM as the first para to builtin functions + args = append(args, v.selfObject()) + } } args = append(args, v.stack[v.sp-numArgs:v.sp]...) ret, e := value.Call(args...) From cc984e86f879f851d03f4460b186fb40f370ffcc Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Thu, 1 Apr 2021 15:07:05 +0000 Subject: [PATCH 07/20] goroutineVM: block parent VM untill all descendant VMs exit The goroutineVM will not exit unless: 1. All its descendant VMs exit 2. It calls abort() 3. Its goroutineVM object abort() is called on behalf of its parent VM The latter 2 cases will trigger aborting procedure of all the descendant VMs, which will further result in #1 above. --- docs/builtins.md | 15 ++++++- errors.go | 3 ++ goroutinevm.go | 64 +++++++++++++++++---------- vm.go | 110 +++++++++++++++++++++++++++-------------------- 4 files changed, 122 insertions(+), 70 deletions(-) diff --git a/docs/builtins.md b/docs/builtins.md index e9044544..4a47ab81 100644 --- a/docs/builtins.md +++ b/docs/builtins.md @@ -128,10 +128,17 @@ items := splice(v, 1, 1, "d", "e") // items == ["b"], v == ["a", "d", "e", "c"] ## govm -Starts a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from +Starts a goroutine which runs fn(arg1, arg2, ...) in a new VM cloned from the current running VM, and returns a goroutineVM object that has wait, result, abort methods. +The goroutineVM will not exit unless: +1. All its descendant VMs exit +2. It calls abort() +3. Its goroutineVM object abort() is called on behalf of its parent VM +The latter 2 cases will trigger aborting procedure of all the descendant VMs, +which will further result in #1 above. + ```golang var := 0 @@ -199,11 +206,15 @@ fmt.println("server: ", gvmServer.result()) returns true if the goroutineVM exited(successfully or not) within the timeout peroid. It waits forever if the optional timeout not specified, or timeout < 0. -* abort() terminates the VM. +* abort() terminates the current VM and all its descendant VMs. * result() waits the goroutineVM to complete, returns Error object if any runtime error occurred during the execution, otherwise returns the result value of fn(arg1, arg2, ...) +## abort +Terminates the current VM and all its descendant VMs. Calling abort() will +always result the current VM returns ErrVMAborted. + ## makechan Makes a channel to send/receive object and returns a chan object that has diff --git a/errors.go b/errors.go index 8ef610a3..5c66cc08 100644 --- a/errors.go +++ b/errors.go @@ -52,6 +52,9 @@ var ( // ErrInvalidRangeStep is an error where the step parameter is less than or equal to 0 when using builtin range function. ErrInvalidRangeStep = errors.New("range step must be greater than 0") + + // ErrVMAborted is an error to denote the VM was forcibly terminated without proper exit. + ErrVMAborted = errors.New("virtual machine aborted") ) // ErrInvalidArgumentType represents an invalid argument value type error. diff --git a/goroutinevm.go b/goroutinevm.go index 69b8c876..fcb68c66 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -7,6 +7,7 @@ import ( func init() { addBuiltinFunction("govm", builtinGovm, true) + addBuiltinFunction("abort", builtinAbort, true) addBuiltinFunction("makechan", builtinMakechan, false) } @@ -22,8 +23,15 @@ type goroutineVM struct { done int64 } -// Start a goroutine which run fn(arg1, arg2, ...) in a new VM cloned from the current running VM. -// Return a goroutineVM object that has wait, result, abort methods. +// Starts a goroutine which runs fn(arg1, arg2, ...) in a new VM cloned from the current running VM. +// Returns a goroutineVM object that has wait, result, abort methods. +// +// The goroutineVM will not exit unless: +// 1. All its descendant VMs exit +// 2. It calls abort() +// 3. Its goroutineVM object abort() is called on behalf of its parent VM +// The latter 2 cases will trigger aborting procedure of all the descendant VMs, which will +// further result in #1 above. func builtinGovm(args ...Object) (Object, error) { vm := args[0].(*vmObj).Value args = args[1:] // the first arg is vmObj inserted by VM @@ -42,12 +50,14 @@ func builtinGovm(args ...Object) (Object, error) { newVM := vm.ShallowClone() gvm := &goroutineVM{ VM: newVM, - waitChan: make(chan ret), + waitChan: make(chan ret, 1), } + vm.addChildVM(gvm.VM) go func() { val, err := gvm.RunCompiled(fn, args[1:]...) gvm.waitChan <- ret{val, err} + vm.delChildVM(gvm.VM) }() obj := map[string]Object{ @@ -58,7 +68,19 @@ func builtinGovm(args ...Object) (Object, error) { return &Map{Value: obj}, nil } -// Return true if the goroutineVM is done +// Terminates the current VM and all its descendant VMs. +// Calling abort() will always result the current VM returns ErrVMAborted. +func builtinAbort(args ...Object) (Object, error) { + vm := args[0].(*vmObj).Value + args = args[1:] // the first arg is vmObj inserted by VM + if len(args) != 0 { + return nil, ErrWrongNumArguments + } + vm.Abort() // aborts self and all descendant VMs + return nil, nil +} + +// Returns true if the goroutineVM is done func (gvm *goroutineVM) wait(seconds int64) bool { if atomic.LoadInt64(&gvm.done) == 1 { return true @@ -78,9 +100,9 @@ func (gvm *goroutineVM) wait(seconds int64) bool { return true } -// Wait for the goroutineVM to complete in timeout seconds. -// Return true if the goroutineVM exited(successfully or not) within the timeout peroid. -// Wait forever if the optional timeout not specified, or timeout < 0. +// Waits for the goroutineVM to complete in timeout seconds. +// Returns true if the goroutineVM exited(successfully or not) within the timeout peroid. +// Waits forever if the optional timeout not specified, or timeout < 0. func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { if len(args) > 1 { return nil, ErrWrongNumArguments @@ -104,7 +126,7 @@ func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { return FalseValue, nil } -// Terminate the execution of the goroutineVM. +// Terminates the execution of the current VM and all its descendant VMs. func (gvm *goroutineVM) abort(args ...Object) (Object, error) { if len(args) != 0 { return nil, ErrWrongNumArguments @@ -114,7 +136,7 @@ func (gvm *goroutineVM) abort(args ...Object) (Object, error) { return nil, nil } -// Wait the goroutineVM to complete, return Error object if any runtime error occurred +// Waits the goroutineVM to complete, return Error object if any runtime error occurred // during the execution, otherwise return the result value of fn(arg1, arg2, ...) func (gvm *goroutineVM) getRet(args ...Object) (Object, error) { if len(args) != 0 { @@ -131,8 +153,8 @@ func (gvm *goroutineVM) getRet(args ...Object) (Object, error) { type objchan chan Object -// Make a channel to send/receive object -// Return a chan object that has send, recv, close methods. +// Makes a channel to send/receive object +// Returns a chan object that has send, recv, close methods. func builtinMakechan(args ...Object) (Object, error) { var size int switch len(args) { @@ -160,8 +182,8 @@ func builtinMakechan(args ...Object) (Object, error) { return &Map{Value: obj}, nil } -// Send an obj to the channel, will block until channel is not full or (*VM).Abort() has been called. -// Send to a closed channel causes panic. +// Sends an obj to the channel, will block if channel is full and the VM has not been aborted. +// Sends to a closed channel causes panic. func (oc objchan) send(args ...Object) (Object, error) { vm := args[0].(*vmObj).Value args = args[1:] // the first arg is vmObj inserted by VM @@ -169,16 +191,15 @@ func (oc objchan) send(args ...Object) (Object, error) { return nil, ErrWrongNumArguments } select { - case <-vm.ctx.Done(): - return nil, vm.ctx.Err() - //return &String{Value: vm.ctx.Err().Error()}, nil + case <-vm.abortChan: + return nil, ErrVMAborted case oc <- args[0]: } return nil, nil } -// Receive an obj from the channel, will block until channel is not empty or (*VM).Abort() has been called. -// Receive from a closed channel returns undefined value. +// Receives an obj from the channel, will block if channel is empty and the VM has not been aborted. +// Receives from a closed channel returns undefined value. func (oc objchan) recv(args ...Object) (Object, error) { vm := args[0].(*vmObj).Value args = args[1:] // the first arg is vmObj inserted by VM @@ -186,9 +207,8 @@ func (oc objchan) recv(args ...Object) (Object, error) { return nil, ErrWrongNumArguments } select { - case <-vm.ctx.Done(): - return nil, vm.ctx.Err() - //return &String{Value: vm.ctx.Err().Error()}, nil + case <-vm.abortChan: + return nil, ErrVMAborted case obj, ok := <-oc: if ok { return obj, nil @@ -197,7 +217,7 @@ func (oc objchan) recv(args ...Object) (Object, error) { return nil, nil } -// Close the channel. +// Closes the channel. func (oc objchan) close(args ...Object) (Object, error) { if len(args) != 0 { return nil, ErrWrongNumArguments diff --git a/vm.go b/vm.go index 3a4fd195..6c629eb5 100644 --- a/vm.go +++ b/vm.go @@ -1,8 +1,8 @@ package tengo import ( - "context" "fmt" + "sync" "sync/atomic" "github.com/d5/tengo/v2/parser" @@ -17,14 +17,14 @@ type frame struct { basePointer int } -type cancelCtx struct { - context.Context - cancel context.CancelFunc +type vmLifecycleCtl struct { + sync.WaitGroup + sync.Mutex + vmMap map[*VM]struct{} } // VM is a virtual machine that executes the bytecode compiled by Compiler. type VM struct { - ctx cancelCtx // used to signal blocked channel sending and receiving to exit constants []Object stack [StackSize]Object sp int @@ -39,6 +39,8 @@ type VM struct { maxAllocs int64 allocs int64 err error + abortChan chan struct{} + childCtl vmLifecycleCtl } // NewVM creates a VM. @@ -58,9 +60,9 @@ func NewVM( framesIndex: 1, ip: -1, maxAllocs: maxAllocs, + abortChan: make(chan struct{}), + childCtl: vmLifecycleCtl{vmMap: make(map[*VM]struct{})}, } - ctx, cancel := context.WithCancel(context.Background()) - v.ctx = cancelCtx{ctx, cancel} v.frames[0].fn = bytecode.MainFunction v.frames[0].ip = -1 v.curFrame = &v.frames[0] @@ -68,29 +70,26 @@ func NewVM( return v } -// Abort aborts the execution. +// Abort aborts the execution of current VM and all its descendant VMs. func (v *VM) Abort() { atomic.StoreInt64(&v.aborting, 1) - v.ctx.cancel() + close(v.abortChan) // broadcast to all receivers + v.childCtl.Lock() + for cvm := range v.childCtl.vmMap { + cvm.Abort() + } + v.childCtl.Unlock() } // Run starts the execution. -func (v *VM) Run() error { - // reset VM states - v.sp = 0 - v.curFrame = &(v.frames[0]) - v.curInsts = v.curFrame.fn.Instructions - v.framesIndex = 1 - v.ip = -1 - v.allocs = v.maxAllocs + 1 - - v.run() - return v.postRun() +func (v *VM) Run() (err error) { + _, err = v.RunCompiled(nil) + return } func (v *VM) postRun() (err error) { err = v.err - if err != nil { + if err != nil && err != ErrVMAborted { filePos := v.fileSet.Position( v.curFrame.fn.SourcePos(v.ip - 1)) err = fmt.Errorf("Runtime Error: %w\n\tat %s", @@ -102,17 +101,13 @@ func (v *VM) postRun() (err error) { v.curFrame.fn.SourcePos(v.curFrame.ip - 1)) err = fmt.Errorf("%w\n\tat %s", err, filePos) } + return } if atomic.LoadInt64(&v.aborting) == 1 { - if err != nil { - err = fmt.Errorf("VM aborted\n\t%w", err) - } else { - err = fmt.Errorf("VM aborted") - } + err = ErrVMAborted } - atomic.StoreInt64(&v.aborting, 0) - return err + return } func concatInsts(instructions ...[]byte) []byte { @@ -139,11 +134,10 @@ func (v *VM) ShallowClone() *VM { framesIndex: 1, ip: -1, maxAllocs: v.maxAllocs, + abortChan: make(chan struct{}), + childCtl: vmLifecycleCtl{vmMap: make(map[*VM]struct{})}, } - //ctx, cancel := context.WithCancel(v.ctx.Context) // cloned VM derives the context of its parent - ctx, cancel := context.WithCancel(context.Background()) // cloned VM has independent context - shallowClone.ctx = cancelCtx{ctx, cancel} // set to empty entry shallowClone.frames[0].fn = emptyEntry shallowClone.frames[0].ip = -1 @@ -166,35 +160,59 @@ var funcWrapper = &CompiledFunction{ VarArgs: true, } +func (v *VM) addChildVM(cvm *VM) { + v.childCtl.Add(1) + v.childCtl.Lock() + v.childCtl.vmMap[cvm] = struct{}{} + v.childCtl.Unlock() +} + +func (v *VM) delChildVM(cvm *VM) { + v.childCtl.Lock() + delete(v.childCtl.vmMap, cvm) + v.childCtl.Unlock() + v.childCtl.Done() +} + // RunCompiled run the VM with user supplied function fn. func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (Object, error) { - entry := &CompiledFunction{ - Instructions: concatInsts( - MakeInstruction(parser.OpCall, 1+len(args), 0), - MakeInstruction(parser.OpSuspend), - ), - } - - v.stack[0] = funcWrapper - v.stack[1] = fn - for i, arg := range args { - v.stack[i+2] = arg + if fn == nil { // normal Run + // reset VM states + v.sp = 0 + } else { // run user supplied function + entry := &CompiledFunction{ + Instructions: concatInsts( + MakeInstruction(parser.OpCall, 1+len(args), 0), + MakeInstruction(parser.OpSuspend), + ), + } + v.stack[0] = funcWrapper + v.stack[1] = fn + for i, arg := range args { + v.stack[i+2] = arg + } + v.sp = 2 + len(args) + v.frames[0].fn = entry + v.frames[0].ip = -1 } - v.sp = 2 + len(args) - v.frames[0].fn = entry - v.frames[0].ip = -1 v.curFrame = &v.frames[0] v.curInsts = v.curFrame.fn.Instructions v.framesIndex = 1 v.ip = -1 v.allocs = v.maxAllocs + 1 + atomic.StoreInt64(&v.aborting, 0) v.run() + v.childCtl.Wait() // waits for all child VMs to exit + if err := v.postRun(); err != nil { return UndefinedValue, err } - return v.stack[v.sp-1], nil + if fn != nil { + return v.stack[v.sp-1], nil + } + return nil, nil } type vmObj struct { From 5047ab73e8da7c7ed700afc3b27cbfc07a8b008f Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Wed, 14 Apr 2021 16:46:52 +0000 Subject: [PATCH 08/20] correct source position for closure function in call stack Take below snipet as an example: $ cat cmd/tengo/test.tengo test1 := func() { v := 1 test2 := func() { len(1,1,1,1) v++ } test2() } test1() ---- before the fix: $ ./tengo test.tengo Runtime Error: wrong number of arguments in call to 'builtin-function:len' at - at test.tengo:7:2 at test.tengo:10:1 ---- after the fix: $ ./tengo test.tengo Runtime Error: wrong number of arguments in call to 'builtin-function:len' at test.tengo:4:3 at test.tengo:7:2 at test.tengo:10:1 --- objects.go | 1 + vm.go | 1 + 2 files changed, 2 insertions(+) diff --git a/objects.go b/objects.go index 416dcfc7..8f5953ec 100644 --- a/objects.go +++ b/objects.go @@ -595,6 +595,7 @@ func (o *CompiledFunction) Copy() Object { NumLocals: o.NumLocals, NumParameters: o.NumParameters, VarArgs: o.VarArgs, + SourceMap: o.SourceMap, Free: append([]*ObjectPtr{}, o.Free...), // DO NOT Copy() of elements; these are variable pointers } } diff --git a/vm.go b/vm.go index 6c629eb5..bda543a6 100644 --- a/vm.go +++ b/vm.go @@ -903,6 +903,7 @@ func (v *VM) run() { NumLocals: fn.NumLocals, NumParameters: fn.NumParameters, VarArgs: fn.VarArgs, + SourceMap: fn.SourceMap, Free: free, } v.allocs-- From c4aa6ae3bb14d61df697ba6542846998fcdd7f30 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Wed, 14 Apr 2021 17:07:04 +0000 Subject: [PATCH 09/20] govm now supports calling both script functions and native functions with this commit: 1. builtinGovm() now accepts native golang function, but unlike compiled functions which run in a cloned VM, native golang functions do not need VM, they just run in a goroutine. 2. Parent VMs now have records of errors of all its descendent VMs. A VM's error is a combination of the error of itself and all errors of the descendent VMs, in another word, parent VM's error includes all errors of the child VMs. 3. ErrVMAborted is no longer treated as runtime error. --- docs/builtins.md | 10 ++----- goroutinevm.go | 50 +++++++++++++++++++++++---------- vm.go | 73 +++++++++++++++++++++++++++++++++++------------- 3 files changed, 91 insertions(+), 42 deletions(-) diff --git a/docs/builtins.md b/docs/builtins.md index 4a47ab81..500721bb 100644 --- a/docs/builtins.md +++ b/docs/builtins.md @@ -190,30 +190,24 @@ if ok := gvmClient.wait(5); !ok { } gvmServer.abort() -fmt.println("client: ", gvmClient.result()) -fmt.println("server: ", gvmServer.result()) - //output: //hello //world //100 //101 -//client: error: "VM aborted" -//server: error: "VM aborted\n\tRuntime Error: context canceled at -\n\tat -" ``` * wait() waits for the goroutineVM to complete in timeout seconds and returns true if the goroutineVM exited(successfully or not) within the timeout peroid. It waits forever if the optional timeout not specified, or timeout < 0. -* abort() terminates the current VM and all its descendant VMs. +* abort() terminates the goroutineVM and all its descendant VMs. * result() waits the goroutineVM to complete, returns Error object if any runtime error occurred during the execution, otherwise returns the result value of fn(arg1, arg2, ...) ## abort -Terminates the current VM and all its descendant VMs. Calling abort() will -always result the current VM returns ErrVMAborted. +Terminates the current VM and all its descendant VMs. ## makechan diff --git a/goroutinevm.go b/goroutinevm.go index fcb68c66..b0de282b 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -17,8 +17,8 @@ type ret struct { } type goroutineVM struct { - *VM - ret // return value of (*VM).RunCompiled() + *VM // if not nil, run CompiledFunction in VM + ret // return value waitChan chan ret done int64 } @@ -38,26 +38,46 @@ func builtinGovm(args ...Object) (Object, error) { if len(args) == 0 { return nil, ErrWrongNumArguments } - fn, ok := args[0].(*CompiledFunction) - if !ok { + + fn := args[0] + if !fn.CanCall() { return nil, ErrInvalidArgumentType{ Name: "first", - Expected: "func", - Found: args[0].TypeName(), + Expected: "callable function", + Found: fn.TypeName(), } } - newVM := vm.ShallowClone() gvm := &goroutineVM{ - VM: newVM, waitChan: make(chan ret, 1), } + cfn, compiled := fn.(*CompiledFunction) + if compiled { + gvm.VM = vm.ShallowClone() + } - vm.addChildVM(gvm.VM) + vm.addChild(gvm.VM) go func() { - val, err := gvm.RunCompiled(fn, args[1:]...) + var val Object + var err error + if cfn != nil { + val, err = gvm.RunCompiled(cfn, args[1:]...) + } else { + var nargs []Object + if bltnfn, ok := fn.(*BuiltinFunction); ok { + if bltnfn.needvmObj { + // pass VM as the first para to builtin functions + nargs = append(nargs, vm.selfObject()) + } + } + nargs = append(nargs, args[1:]...) + val, err = fn.Call(nargs...) + } + if err != nil { + vm.addError(err) + } gvm.waitChan <- ret{val, err} - vm.delChildVM(gvm.VM) + vm.delChild(gvm.VM) }() obj := map[string]Object{ @@ -69,7 +89,6 @@ func builtinGovm(args ...Object) (Object, error) { } // Terminates the current VM and all its descendant VMs. -// Calling abort() will always result the current VM returns ErrVMAborted. func builtinAbort(args ...Object) (Object, error) { vm := args[0].(*vmObj).Value args = args[1:] // the first arg is vmObj inserted by VM @@ -126,13 +145,14 @@ func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { return FalseValue, nil } -// Terminates the execution of the current VM and all its descendant VMs. +// Terminates the execution of the goroutineVM and all its descendant VMs. func (gvm *goroutineVM) abort(args ...Object) (Object, error) { if len(args) != 0 { return nil, ErrWrongNumArguments } - - gvm.Abort() + if gvm.VM != nil { + gvm.Abort() + } return nil, nil } diff --git a/vm.go b/vm.go index bda543a6..bfd5775d 100644 --- a/vm.go +++ b/vm.go @@ -1,7 +1,9 @@ package tengo import ( + "errors" "fmt" + "strings" "sync" "sync/atomic" @@ -17,10 +19,11 @@ type frame struct { basePointer int } -type vmLifecycleCtl struct { +type vmChildCtl struct { sync.WaitGroup sync.Mutex - vmMap map[*VM]struct{} + vmMap map[*VM]struct{} + errors []error } // VM is a virtual machine that executes the bytecode compiled by Compiler. @@ -40,7 +43,7 @@ type VM struct { allocs int64 err error abortChan chan struct{} - childCtl vmLifecycleCtl + childCtl vmChildCtl } // NewVM creates a VM. @@ -61,7 +64,7 @@ func NewVM( ip: -1, maxAllocs: maxAllocs, abortChan: make(chan struct{}), - childCtl: vmLifecycleCtl{vmMap: make(map[*VM]struct{})}, + childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, } v.frames[0].fn = bytecode.MainFunction v.frames[0].ip = -1 @@ -87,9 +90,33 @@ func (v *VM) Run() (err error) { return } +type vmError struct { + self error + children []error +} + +func (vme vmError) Error() string { + var b strings.Builder + if vme.self != nil { + fmt.Fprintf(&b, "%v\n", vme.self) + } + for _, err := range vme.children { + for _, s := range strings.Split(err.Error(), "\n") { + if len(s) != 0 && s != "\tat -" { + fmt.Fprintf(&b, "%s\n", s) + } + } + } + return b.String() +} + func (v *VM) postRun() (err error) { err = v.err - if err != nil && err != ErrVMAborted { + // ErrVMAborted is user behavior thus it is not an actual runtime error + if errors.Is(err, ErrVMAborted) { + err = nil + } + if err != nil { filePos := v.fileSet.Position( v.curFrame.fn.SourcePos(v.ip - 1)) err = fmt.Errorf("Runtime Error: %w\n\tat %s", @@ -101,11 +128,9 @@ func (v *VM) postRun() (err error) { v.curFrame.fn.SourcePos(v.curFrame.ip - 1)) err = fmt.Errorf("%w\n\tat %s", err, filePos) } - return } - - if atomic.LoadInt64(&v.aborting) == 1 { - err = ErrVMAborted + if err != nil || len(v.childCtl.errors) != 0 { + err = vmError{err, v.childCtl.errors} } return } @@ -135,7 +160,7 @@ func (v *VM) ShallowClone() *VM { ip: -1, maxAllocs: v.maxAllocs, abortChan: make(chan struct{}), - childCtl: vmLifecycleCtl{vmMap: make(map[*VM]struct{})}, + childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, } // set to empty entry @@ -160,17 +185,27 @@ var funcWrapper = &CompiledFunction{ VarArgs: true, } -func (v *VM) addChildVM(cvm *VM) { - v.childCtl.Add(1) +func (v *VM) addError(err error) { v.childCtl.Lock() - v.childCtl.vmMap[cvm] = struct{}{} + v.childCtl.errors = append(v.childCtl.errors, err) v.childCtl.Unlock() } -func (v *VM) delChildVM(cvm *VM) { - v.childCtl.Lock() - delete(v.childCtl.vmMap, cvm) - v.childCtl.Unlock() +func (v *VM) addChild(cvm *VM) { + v.childCtl.Add(1) + if cvm != nil { + v.childCtl.Lock() + v.childCtl.vmMap[cvm] = struct{}{} + v.childCtl.Unlock() + } +} + +func (v *VM) delChild(cvm *VM) { + if cvm != nil { + v.childCtl.Lock() + delete(v.childCtl.vmMap, cvm) + v.childCtl.Unlock() + } v.childCtl.Done() } @@ -209,10 +244,10 @@ func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (Object, error) { if err := v.postRun(); err != nil { return UndefinedValue, err } - if fn != nil { + if fn != nil && atomic.LoadInt64(&v.aborting) == 0 { return v.stack[v.sp-1], nil } - return nil, nil + return UndefinedValue, nil } type vmObj struct { From c81780bfa76097a5606a6082a0f53466c578c815 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Wed, 14 Apr 2021 17:56:09 +0000 Subject: [PATCH 10/20] rename builtin govm back to go now that "govm" does not always run function in VM, so "go" is better. update docs to reflect the usage changes. --- docs/builtins.md | 33 +++++++++++++++++++-------------- goroutinevm.go | 23 +++++++++++++++-------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/docs/builtins.md b/docs/builtins.md index 500721bb..523d8007 100644 --- a/docs/builtins.md +++ b/docs/builtins.md @@ -126,18 +126,22 @@ v := ["a", "b", "c"] items := splice(v, 1, 1, "d", "e") // items == ["b"], v == ["a", "d", "e", "c"] ``` -## govm +## go -Starts a goroutine which runs fn(arg1, arg2, ...) in a new VM cloned from -the current running VM, and returns a goroutineVM object that has -wait, result, abort methods. +Starts a independent concurrent goroutine which runs fn(arg1, arg2, ...) + +If fn is CompiledFunction, the current running VM will be cloned to create +a new VM in which the CompiledFunction will be running. +The fn can also be any object that has Call() method, such as BuiltinFunction, +in which case no cloned VM will be created. +Returns a goroutineVM object that has wait, result, abort methods. The goroutineVM will not exit unless: -1. All its descendant VMs exit +1. All its descendant goroutineVMs exit 2. It calls abort() 3. Its goroutineVM object abort() is called on behalf of its parent VM -The latter 2 cases will trigger aborting procedure of all the descendant VMs, -which will further result in #1 above. +The latter 2 cases will trigger aborting procedure of all the descendant +goroutineVMs, which will further result in #1 above. ```golang var := 0 @@ -145,8 +149,8 @@ var := 0 f1 := func(a,b) { var = 10; return a+b } f2 := func(a,b,c) { var = 11; return a+b+c } -gvm1 := govm(f1,1,2) -gvm2 := govm(f2,1,2,5) +gvm1 := go(f1,1,2) +gvm2 := go(f2,1,2,5) fmt.println(gvm1.result()) // 3 fmt.println(gvm2.result()) // 8 @@ -182,8 +186,8 @@ server := func() { } } -gvmClient := govm(client, 2) -gvmServer := govm(server) +gvmClient := go(client, 2) +gvmServer := go(server) if ok := gvmClient.wait(5); !ok { gvmClient.abort() @@ -197,17 +201,18 @@ gvmServer.abort() //101 ``` -* wait() waits for the goroutineVM to complete in timeout seconds and +* wait() waits for the goroutineVM to complete up to timeout seconds and returns true if the goroutineVM exited(successfully or not) within the timeout peroid. It waits forever if the optional timeout not specified, or timeout < 0. -* abort() terminates the goroutineVM and all its descendant VMs. +* abort() triggers the termination process of the goroutineVM and all +its descendant VMs. * result() waits the goroutineVM to complete, returns Error object if any runtime error occurred during the execution, otherwise returns the result value of fn(arg1, arg2, ...) ## abort -Terminates the current VM and all its descendant VMs. +Triggers the termination process of the current VM and all its descendant VMs. ## makechan diff --git a/goroutinevm.go b/goroutinevm.go index b0de282b..4fa0bc40 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -6,7 +6,7 @@ import ( ) func init() { - addBuiltinFunction("govm", builtinGovm, true) + addBuiltinFunction("go", builtinGovm, true) addBuiltinFunction("abort", builtinAbort, true) addBuiltinFunction("makechan", builtinMakechan, false) } @@ -23,15 +23,22 @@ type goroutineVM struct { done int64 } -// Starts a goroutine which runs fn(arg1, arg2, ...) in a new VM cloned from the current running VM. +// Starts a independent concurrent goroutine which runs fn(arg1, arg2, ...) +// +// If fn is CompiledFunction, the current running VM will be cloned to create +// a new VM in which the CompiledFunction will be running. +// +// The fn can also be any object that has Call() method, such as BuiltinFunction, +// in which case no cloned VM will be created. +// // Returns a goroutineVM object that has wait, result, abort methods. // // The goroutineVM will not exit unless: -// 1. All its descendant VMs exit +// 1. All its descendant goroutineVMs exit // 2. It calls abort() // 3. Its goroutineVM object abort() is called on behalf of its parent VM -// The latter 2 cases will trigger aborting procedure of all the descendant VMs, which will -// further result in #1 above. +// The latter 2 cases will trigger aborting procedure of all the descendant goroutineVMs, +// which will further result in #1 above. func builtinGovm(args ...Object) (Object, error) { vm := args[0].(*vmObj).Value args = args[1:] // the first arg is vmObj inserted by VM @@ -88,7 +95,7 @@ func builtinGovm(args ...Object) (Object, error) { return &Map{Value: obj}, nil } -// Terminates the current VM and all its descendant VMs. +// Triggers the termination process of the current VM and all its descendant VMs. func builtinAbort(args ...Object) (Object, error) { vm := args[0].(*vmObj).Value args = args[1:] // the first arg is vmObj inserted by VM @@ -119,7 +126,7 @@ func (gvm *goroutineVM) wait(seconds int64) bool { return true } -// Waits for the goroutineVM to complete in timeout seconds. +// Waits for the goroutineVM to complete up to timeout seconds. // Returns true if the goroutineVM exited(successfully or not) within the timeout peroid. // Waits forever if the optional timeout not specified, or timeout < 0. func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { @@ -145,7 +152,7 @@ func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { return FalseValue, nil } -// Terminates the execution of the goroutineVM and all its descendant VMs. +// Triggers the termination process of the goroutineVM and all its descendant VMs. func (gvm *goroutineVM) abort(args ...Object) (Object, error) { if len(args) != 0 { return nil, ErrWrongNumArguments From cf3fef9229712aa7cc91bb962411780fd835aea2 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Thu, 15 Apr 2021 16:09:02 +0000 Subject: [PATCH 11/20] Wrap main VM's error to make TestVMErrorUnwrap test pass And now abort() on the same VM twice will not panic. --- vm.go | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/vm.go b/vm.go index bfd5775d..0f6facb0 100644 --- a/vm.go +++ b/vm.go @@ -75,6 +75,9 @@ func NewVM( // Abort aborts the execution of current VM and all its descendant VMs. func (v *VM) Abort() { + if atomic.LoadInt64(&v.aborting) != 0 { + return + } atomic.StoreInt64(&v.aborting, 1) close(v.abortChan) // broadcast to all receivers v.childCtl.Lock() @@ -90,26 +93,6 @@ func (v *VM) Run() (err error) { return } -type vmError struct { - self error - children []error -} - -func (vme vmError) Error() string { - var b strings.Builder - if vme.self != nil { - fmt.Fprintf(&b, "%v\n", vme.self) - } - for _, err := range vme.children { - for _, s := range strings.Split(err.Error(), "\n") { - if len(s) != 0 && s != "\tat -" { - fmt.Fprintf(&b, "%s\n", s) - } - } - } - return b.String() -} - func (v *VM) postRun() (err error) { err = v.err // ErrVMAborted is user behavior thus it is not an actual runtime error @@ -129,8 +112,19 @@ func (v *VM) postRun() (err error) { err = fmt.Errorf("%w\n\tat %s", err, filePos) } } - if err != nil || len(v.childCtl.errors) != 0 { - err = vmError{err, v.childCtl.errors} + + var sb strings.Builder + for _, cerr := range v.childCtl.errors { + fmt.Fprintf(&sb, "%v\n", cerr) + } + cerrs := sb.String() + + if err != nil && len(cerrs) != 0 { + err = fmt.Errorf("%w\n%s", err, cerrs) + return + } + if len(cerrs) != 0 { + err = fmt.Errorf("%s", cerrs) } return } From 4462ad92b4524350797457dce069d105b8d49e05 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Thu, 15 Apr 2021 17:28:57 +0000 Subject: [PATCH 12/20] update docs for builtin go --- docs/builtins.md | 156 +++++++++++++++++++++++++++++++++++++++++------ goroutinevm.go | 2 +- 2 files changed, 139 insertions(+), 19 deletions(-) diff --git a/docs/builtins.md b/docs/builtins.md index 523d8007..63aae5d8 100644 --- a/docs/builtins.md +++ b/docs/builtins.md @@ -128,7 +128,7 @@ items := splice(v, 1, 1, "d", "e") // items == ["b"], v == ["a", "d", "e", "c"] ## go -Starts a independent concurrent goroutine which runs fn(arg1, arg2, ...) +Starts an independent concurrent goroutine which runs fn(arg1, arg2, ...) If fn is CompiledFunction, the current running VM will be cloned to create a new VM in which the CompiledFunction will be running. @@ -157,6 +157,18 @@ fmt.println(gvm2.result()) // 8 fmt.println(var) // 10 or 11 ``` +* wait() waits for the goroutineVM to complete up to timeout seconds and +returns true if the goroutineVM exited(successfully or not) within the +timeout. It waits forever if the optional timeout not specified, +or timeout < 0. +* abort() triggers the termination process of the goroutineVM and all +its descendant VMs. +* result() waits the goroutineVM to complete, returns Error object if +any runtime error occurred during the execution, otherwise returns the +result value of fn(arg1, arg2, ...) + +### 1 client 1 server + Below is a simple client server example: ```golang @@ -165,12 +177,10 @@ repChan := makechan(8) client := func(interval) { reqChan.send("hello") - i := 0 - for { + for i := 0; true; i++ { fmt.println(repChan.recv()) times.sleep(interval*times.second) reqChan.send(i) - i++ } } @@ -186,13 +196,13 @@ server := func() { } } -gvmClient := go(client, 2) -gvmServer := go(server) +gClient := go(client, 2) +gServer := go(server) -if ok := gvmClient.wait(5); !ok { - gvmClient.abort() +if ok := gClient.wait(5); !ok { + gClient.abort() } -gvmServer.abort() +gServer.abort() //output: //hello @@ -201,15 +211,125 @@ gvmServer.abort() //101 ``` -* wait() waits for the goroutineVM to complete up to timeout seconds and -returns true if the goroutineVM exited(successfully or not) within the -timeout peroid. It waits forever if the optional timeout not specified, -or timeout < 0. -* abort() triggers the termination process of the goroutineVM and all -its descendant VMs. -* result() waits the goroutineVM to complete, returns Error object if -any runtime error occurred during the execution, otherwise returns the -result value of fn(arg1, arg2, ...) +### n client n server, channel in channel + +```golang +sharedReqChan := makechan(128) + +client = func(name, interval, timeout) { + print := func(s) { + fmt.println(name, s) + } + print("started") + + repChan := makechan(1) + msg := {chan:repChan} + + msg.data = "hello" + sharedReqChan.send(msg) + print(repChan.recv()) + + for i := 0; i * interval < timeout; i++ { + msg.data = i + sharedReqChan.send(msg) + print(repChan.recv()) + times.sleep(interval*times.second) + } +} + +server = func(name) { + print := func(s) { + fmt.println(name, s) + } + print("started") + + for { + req := sharedReqChan.recv() + if req.data == "hello" { + req.chan.send("world") + } else { + req.chan.send(req.data+100) + } + } +} + +clients := func() { + for i :=0; i < 5; i++ { + go(client, format("client %d: ", i), 1, 4) + } +} + +servers := func() { + for i :=0; i < 2; i++ { + go(server, format("server %d: ", i)) + } +} + +// After 4 seconds, all clients should have exited normally +gclts := go(clients) +// If servers exit earlier than clients, then clients may be +// blocked forever waiting for the reply chan, because servers +// were aborted with the req fetched from sharedReqChan before +// sending back the reply. +// In such case, do below to abort() the clients manually +//go(func(){times.sleep(6*times.second); gclts.abort()}) + +// Servers are infinite loop, abort() them after 5 seconds +gsrvs := go(servers) +if ok := gsrvs.wait(5); !ok { + gsrvs.abort() +} + +// Main VM waits here until all the child "go" finish + +// If somehow the main VM is stuck, that is because there is +// at least one child VM that has not exited as expected, we +// can do abort() to force exit. +abort() + +//output: +//3 +//8 +//hello +//world +//100 +//101 + +//unordered output: +//client 4: started +//server 0: started +//client 4: world +//client 4: 100 +//client 3: started +//client 3: world +//client 3: 100 +//client 2: started +//client 2: world +//client 2: 100 +//client 0: started +//client 0: world +//client 0: 100 +//client 1: started +//client 1: world +//client 1: 100 +//server 1: started +//client 1: 101 +//client 2: 101 +//client 4: 101 +//client 0: 101 +//client 3: 101 +//client 3: 102 +//client 0: 102 +//client 2: 102 +//client 1: 102 +//client 4: 102 +//client 0: 103 +//client 3: 103 +//client 2: 103 +//client 1: 103 +//client 4: 103 + +``` ## abort Triggers the termination process of the current VM and all its descendant VMs. diff --git a/goroutinevm.go b/goroutinevm.go index 4fa0bc40..db8d1039 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -127,7 +127,7 @@ func (gvm *goroutineVM) wait(seconds int64) bool { } // Waits for the goroutineVM to complete up to timeout seconds. -// Returns true if the goroutineVM exited(successfully or not) within the timeout peroid. +// Returns true if the goroutineVM exited(successfully or not) within the timeout. // Waits forever if the optional timeout not specified, or timeout < 0. func (gvm *goroutineVM) waitTimeout(args ...Object) (Object, error) { if len(args) > 1 { From a6205f4369c65ad021d02f4628e01a27df40c9ea Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Sat, 17 Apr 2021 05:49:39 +0000 Subject: [PATCH 13/20] improve panic handling On a panic happens in a VM: * the panic value along with both its script level and native level callstack are recorded as run time error, its parent VM should examine this error. * the panic will not cause whole program exits, but it does trigger the abort chain, so that the VM in which the panic happens and all its descendent VMs will terminate. So panic only causes the descendent group of VMs to exit. Other VM groups are not affected. --- goroutinevm.go | 27 ++++++-- vm.go | 172 ++++++++++++++++++++++++++++++------------------- 2 files changed, 126 insertions(+), 73 deletions(-) diff --git a/goroutinevm.go b/goroutinevm.go index db8d1039..792d4a88 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -1,6 +1,8 @@ package tengo import ( + "fmt" + "runtime/debug" "sync/atomic" "time" ) @@ -25,7 +27,7 @@ type goroutineVM struct { // Starts a independent concurrent goroutine which runs fn(arg1, arg2, ...) // -// If fn is CompiledFunction, the current running VM will be cloned to create +// If fn is CompiledFunction, the current running VM will be cloned to create // a new VM in which the CompiledFunction will be running. // // The fn can also be any object that has Call() method, such as BuiltinFunction, @@ -58,15 +60,33 @@ func builtinGovm(args ...Object) (Object, error) { gvm := &goroutineVM{ waitChan: make(chan ret, 1), } + + var callers []frame cfn, compiled := fn.(*CompiledFunction) if compiled { gvm.VM = vm.ShallowClone() + } else { + callers = vm.callers() } vm.addChild(gvm.VM) go func() { var val Object var err error + defer func() { + if perr := recover(); perr != nil { + if callers == nil { + panic("callers not saved") + } + err = fmt.Errorf("\nRuntime Panic: %v%s\n%s", perr, vm.callStack(callers), debug.Stack()) + } + if err != nil { + vm.addError(err) + } + gvm.waitChan <- ret{val, err} + vm.delChild(gvm.VM) + }() + if cfn != nil { val, err = gvm.RunCompiled(cfn, args[1:]...) } else { @@ -80,11 +100,6 @@ func builtinGovm(args ...Object) (Object, error) { nargs = append(nargs, args[1:]...) val, err = fn.Call(nargs...) } - if err != nil { - vm.addError(err) - } - gvm.waitChan <- ret{val, err} - vm.delChild(gvm.VM) }() obj := map[string]Object{ diff --git a/vm.go b/vm.go index 0f6facb0..a0b310a7 100644 --- a/vm.go +++ b/vm.go @@ -3,6 +3,7 @@ package tengo import ( "errors" "fmt" + "runtime/debug" "strings" "sync" "sync/atomic" @@ -93,42 +94,6 @@ func (v *VM) Run() (err error) { return } -func (v *VM) postRun() (err error) { - err = v.err - // ErrVMAborted is user behavior thus it is not an actual runtime error - if errors.Is(err, ErrVMAborted) { - err = nil - } - if err != nil { - filePos := v.fileSet.Position( - v.curFrame.fn.SourcePos(v.ip - 1)) - err = fmt.Errorf("Runtime Error: %w\n\tat %s", - err, filePos) - for v.framesIndex > 1 { - v.framesIndex-- - v.curFrame = &v.frames[v.framesIndex-1] - filePos = v.fileSet.Position( - v.curFrame.fn.SourcePos(v.curFrame.ip - 1)) - err = fmt.Errorf("%w\n\tat %s", err, filePos) - } - } - - var sb strings.Builder - for _, cerr := range v.childCtl.errors { - fmt.Fprintf(&sb, "%v\n", cerr) - } - cerrs := sb.String() - - if err != nil && len(cerrs) != 0 { - err = fmt.Errorf("%w\n%s", err, cerrs) - return - } - if len(cerrs) != 0 { - err = fmt.Errorf("%s", cerrs) - } - return -} - func concatInsts(instructions ...[]byte) []byte { var concat []byte for _, i := range instructions { @@ -179,32 +144,8 @@ var funcWrapper = &CompiledFunction{ VarArgs: true, } -func (v *VM) addError(err error) { - v.childCtl.Lock() - v.childCtl.errors = append(v.childCtl.errors, err) - v.childCtl.Unlock() -} - -func (v *VM) addChild(cvm *VM) { - v.childCtl.Add(1) - if cvm != nil { - v.childCtl.Lock() - v.childCtl.vmMap[cvm] = struct{}{} - v.childCtl.Unlock() - } -} - -func (v *VM) delChild(cvm *VM) { - if cvm != nil { - v.childCtl.Lock() - delete(v.childCtl.vmMap, cvm) - v.childCtl.Unlock() - } - v.childCtl.Done() -} - // RunCompiled run the VM with user supplied function fn. -func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (Object, error) { +func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (val Object, err error) { if fn == nil { // normal Run // reset VM states v.sp = 0 @@ -232,16 +173,113 @@ func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (Object, error) { v.allocs = v.maxAllocs + 1 atomic.StoreInt64(&v.aborting, 0) + + defer func() { + if perr := recover(); perr != nil { + v.err = ErrPanic{perr, debug.Stack()} + v.Abort() // run time panic should trigger abort chain + } + v.childCtl.Wait() // waits for all child VMs to exit + if err = v.postRun(); err != nil { + return + } + if fn != nil && atomic.LoadInt64(&v.aborting) == 0 { + val = v.stack[v.sp-1] + } + }() + + val = UndefinedValue v.run() - v.childCtl.Wait() // waits for all child VMs to exit + return +} - if err := v.postRun(); err != nil { - return UndefinedValue, err +// ErrPanic is an error where panic happended in the VM. +type ErrPanic struct { + perr interface{} + stack []byte +} + +func (e ErrPanic) Error() string { + return fmt.Sprintf("panic: %v\n%s", e.perr, e.stack) +} + +func (v *VM) addError(err error) { + v.childCtl.Lock() + v.childCtl.errors = append(v.childCtl.errors, err) + v.childCtl.Unlock() +} + +func (v *VM) addChild(cvm *VM) { + v.childCtl.Add(1) + if cvm != nil { + v.childCtl.Lock() + v.childCtl.vmMap[cvm] = struct{}{} + v.childCtl.Unlock() } - if fn != nil && atomic.LoadInt64(&v.aborting) == 0 { - return v.stack[v.sp-1], nil +} + +func (v *VM) delChild(cvm *VM) { + if cvm != nil { + v.childCtl.Lock() + delete(v.childCtl.vmMap, cvm) + v.childCtl.Unlock() + } + v.childCtl.Done() +} + +func (v *VM) callers() (frames []frame) { + curFrame := *v.curFrame + curFrame.ip = v.ip - 1 + frames = append(frames, curFrame) + for i := v.framesIndex - 1; i >= 1; i-- { + curFrame = v.frames[i-1] + frames = append(frames, curFrame) + } + return frames +} + +func (v *VM) callStack(frames []frame) string { + if frames == nil { + frames = v.callers() } - return UndefinedValue, nil + + var sb strings.Builder + for _, f := range frames { + filePos := v.fileSet.Position(f.fn.SourcePos(f.ip)) + fmt.Fprintf(&sb, "\n\tat %s", filePos) + } + return sb.String() +} + +func (v *VM) postRun() (err error) { + err = v.err + // ErrVMAborted is user behavior thus it is not an actual runtime error + if errors.Is(err, ErrVMAborted) { + err = nil + } + if err != nil { + var e ErrPanic + if errors.As(err, &e) { + err = fmt.Errorf("\nRuntime Panic: %v%s\n%s", e.perr, v.callStack(nil), e.stack) + } else { + err = fmt.Errorf("\nRuntime Error: %w%s", err, v.callStack(nil)) + } + } + + var sb strings.Builder + for _, cerr := range v.childCtl.errors { + fmt.Fprintf(&sb, "%v\n", cerr) + } + cerrs := sb.String() + + if err != nil && len(cerrs) != 0 { + err = fmt.Errorf("%w\n%s", err, cerrs) + return + } + if len(cerrs) != 0 { + err = fmt.Errorf("%s", cerrs) + } + return } type vmObj struct { From 496574d7f8f63974c7846312f0a1635ae6a34114 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Mon, 3 May 2021 04:29:09 +0000 Subject: [PATCH 14/20] Make VM's input/output redirectable Export VMObj to std.fmt, so that fmtPrintln can get vm.Out to output to a io.Writer per VM. --- builtins.go | 6 +++--- bytecode.go | 1 + goroutinevm.go | 22 +++++++++++----------- objects.go | 4 ++-- stdlib/fmt.go | 20 +++++++++++++------- vm.go | 15 ++++++++++++--- 6 files changed, 42 insertions(+), 26 deletions(-) diff --git a/builtins.go b/builtins.go index b90da6c4..aa254785 100644 --- a/builtins.go +++ b/builtins.go @@ -2,9 +2,9 @@ package tengo var builtinFuncs []*BuiltinFunction -// if needvmObj is true, VM will pass [vmObj, args...] to fn when calling it. -func addBuiltinFunction(name string, fn CallableFunc, needvmObj bool) { - builtinFuncs = append(builtinFuncs, &BuiltinFunction{Name: name, Value: fn, needvmObj: needvmObj}) +// if needVMObj is true, VM will pass [VMObj, args...] to fn when calling it. +func addBuiltinFunction(name string, fn CallableFunc, needVMObj bool) { + builtinFuncs = append(builtinFuncs, &BuiltinFunction{Name: name, Value: fn, NeedVMObj: needVMObj}) } func init() { diff --git a/bytecode.go b/bytecode.go index f3049cee..02a822bd 100644 --- a/bytecode.go +++ b/bytecode.go @@ -295,4 +295,5 @@ func init() { gob.Register(&Time{}) gob.Register(&Undefined{}) gob.Register(&UserFunction{}) + gob.Register(&BuiltinFunction{}) } diff --git a/goroutinevm.go b/goroutinevm.go index 792d4a88..52826e3d 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -42,8 +42,8 @@ type goroutineVM struct { // The latter 2 cases will trigger aborting procedure of all the descendant goroutineVMs, // which will further result in #1 above. func builtinGovm(args ...Object) (Object, error) { - vm := args[0].(*vmObj).Value - args = args[1:] // the first arg is vmObj inserted by VM + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM if len(args) == 0 { return nil, ErrWrongNumArguments } @@ -92,7 +92,7 @@ func builtinGovm(args ...Object) (Object, error) { } else { var nargs []Object if bltnfn, ok := fn.(*BuiltinFunction); ok { - if bltnfn.needvmObj { + if bltnfn.NeedVMObj { // pass VM as the first para to builtin functions nargs = append(nargs, vm.selfObject()) } @@ -112,8 +112,8 @@ func builtinGovm(args ...Object) (Object, error) { // Triggers the termination process of the current VM and all its descendant VMs. func builtinAbort(args ...Object) (Object, error) { - vm := args[0].(*vmObj).Value - args = args[1:] // the first arg is vmObj inserted by VM + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } @@ -217,8 +217,8 @@ func builtinMakechan(args ...Object) (Object, error) { oc := make(objchan, size) obj := map[string]Object{ - "send": &BuiltinFunction{Value: oc.send, needvmObj: true}, - "recv": &BuiltinFunction{Value: oc.recv, needvmObj: true}, + "send": &BuiltinFunction{Value: oc.send, NeedVMObj: true}, + "recv": &BuiltinFunction{Value: oc.recv, NeedVMObj: true}, "close": &BuiltinFunction{Value: oc.close}, } return &Map{Value: obj}, nil @@ -227,8 +227,8 @@ func builtinMakechan(args ...Object) (Object, error) { // Sends an obj to the channel, will block if channel is full and the VM has not been aborted. // Sends to a closed channel causes panic. func (oc objchan) send(args ...Object) (Object, error) { - vm := args[0].(*vmObj).Value - args = args[1:] // the first arg is vmObj inserted by VM + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { return nil, ErrWrongNumArguments } @@ -243,8 +243,8 @@ func (oc objchan) send(args ...Object) (Object, error) { // Receives an obj from the channel, will block if channel is empty and the VM has not been aborted. // Receives from a closed channel returns undefined value. func (oc objchan) recv(args ...Object) (Object, error) { - vm := args[0].(*vmObj).Value - args = args[1:] // the first arg is vmObj inserted by VM + vm := args[0].(*VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 0 { return nil, ErrWrongNumArguments } diff --git a/objects.go b/objects.go index 8f5953ec..c4fcda7a 100644 --- a/objects.go +++ b/objects.go @@ -322,7 +322,7 @@ type BuiltinFunction struct { ObjectImpl Name string Value CallableFunc - needvmObj bool + NeedVMObj bool } // TypeName returns the name of the type. @@ -336,7 +336,7 @@ func (o *BuiltinFunction) String() string { // Copy returns a copy of the type. func (o *BuiltinFunction) Copy() Object { - return &BuiltinFunction{Value: o.Value} + return &BuiltinFunction{Value: o.Value, NeedVMObj: o.NeedVMObj} } // Equals returns true if the value of the type is equal to the value of diff --git a/stdlib/fmt.go b/stdlib/fmt.go index 9945277f..352c427a 100644 --- a/stdlib/fmt.go +++ b/stdlib/fmt.go @@ -7,22 +7,26 @@ import ( ) var fmtModule = map[string]tengo.Object{ - "print": &tengo.UserFunction{Name: "print", Value: fmtPrint}, - "printf": &tengo.UserFunction{Name: "printf", Value: fmtPrintf}, - "println": &tengo.UserFunction{Name: "println", Value: fmtPrintln}, + "print": &tengo.BuiltinFunction{Value: fmtPrint, NeedVMObj: true}, + "printf": &tengo.BuiltinFunction{Value: fmtPrintf, NeedVMObj: true}, + "println": &tengo.BuiltinFunction{Value: fmtPrintln, NeedVMObj: true}, "sprintf": &tengo.UserFunction{Name: "sprintf", Value: fmtSprintf}, } func fmtPrint(args ...tengo.Object) (ret tengo.Object, err error) { + vm := args[0].(*tengo.VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM printArgs, err := getPrintArgs(args...) if err != nil { return nil, err } - _, _ = fmt.Print(printArgs...) + fmt.Fprint(vm.Out, printArgs...) return nil, nil } func fmtPrintf(args ...tengo.Object) (ret tengo.Object, err error) { + vm := args[0].(*tengo.VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM numArgs := len(args) if numArgs == 0 { return nil, tengo.ErrWrongNumArguments @@ -37,7 +41,7 @@ func fmtPrintf(args ...tengo.Object) (ret tengo.Object, err error) { } } if numArgs == 1 { - fmt.Print(format) + fmt.Fprint(vm.Out, format) return nil, nil } @@ -45,17 +49,19 @@ func fmtPrintf(args ...tengo.Object) (ret tengo.Object, err error) { if err != nil { return nil, err } - fmt.Print(s) + fmt.Fprint(vm.Out, s) return nil, nil } func fmtPrintln(args ...tengo.Object) (ret tengo.Object, err error) { + vm := args[0].(*tengo.VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM printArgs, err := getPrintArgs(args...) if err != nil { return nil, err } printArgs = append(printArgs, "\n") - _, _ = fmt.Print(printArgs...) + fmt.Fprint(vm.Out, printArgs...) return nil, nil } diff --git a/vm.go b/vm.go index a0b310a7..373a6b11 100644 --- a/vm.go +++ b/vm.go @@ -3,6 +3,8 @@ package tengo import ( "errors" "fmt" + "io" + "os" "runtime/debug" "strings" "sync" @@ -45,6 +47,8 @@ type VM struct { err error abortChan chan struct{} childCtl vmChildCtl + In io.Reader + Out io.Writer } // NewVM creates a VM. @@ -66,6 +70,8 @@ func NewVM( maxAllocs: maxAllocs, abortChan: make(chan struct{}), childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, + In: os.Stdin, + Out: os.Stdout, } v.frames[0].fn = bytecode.MainFunction v.frames[0].ip = -1 @@ -120,6 +126,8 @@ func (v *VM) ShallowClone() *VM { maxAllocs: v.maxAllocs, abortChan: make(chan struct{}), childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, + In: v.In, + Out: v.Out, } // set to empty entry @@ -282,13 +290,14 @@ func (v *VM) postRun() (err error) { return } -type vmObj struct { +// VMObj exports VM +type VMObj struct { ObjectImpl Value *VM } func (v *VM) selfObject() Object { - return &vmObj{Value: v} + return &VMObj{Value: v} } func (v *VM) run() { @@ -827,7 +836,7 @@ func (v *VM) run() { } else { var args []Object if bltnfn, ok := value.(*BuiltinFunction); ok { - if bltnfn.needvmObj { + if bltnfn.NeedVMObj { // pass VM as the first para to builtin functions args = append(args, v.selfObject()) } From b6d2abac2c68bbe666c572366d8e867dcbae6526 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Thu, 6 May 2021 17:20:30 +0000 Subject: [PATCH 15/20] Add Args to VM so that each VM can have its own os.Args --- stdlib/os.go | 11 +++++++---- vm.go | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/stdlib/os.go b/stdlib/os.go index 576bc94b..cf3da15b 100644 --- a/stdlib/os.go +++ b/stdlib/os.go @@ -39,9 +39,10 @@ var osModule = map[string]tengo.Object{ "seek_set": &tengo.Int{Value: int64(io.SeekStart)}, "seek_cur": &tengo.Int{Value: int64(io.SeekCurrent)}, "seek_end": &tengo.Int{Value: int64(io.SeekEnd)}, - "args": &tengo.UserFunction{ - Name: "args", - Value: osArgs, + "args": &tengo.BuiltinFunction{ + Name: "args", + Value: osArgs, + NeedVMObj: true, }, // args() => array(string) "chdir": &tengo.UserFunction{ Name: "chdir", @@ -328,11 +329,13 @@ func osOpenFile(args ...tengo.Object) (tengo.Object, error) { } func osArgs(args ...tengo.Object) (tengo.Object, error) { + vm := args[0].(*tengo.VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 0 { return nil, tengo.ErrWrongNumArguments } arr := &tengo.Array{} - for _, osArg := range os.Args { + for _, osArg := range vm.Args { if len(osArg) > tengo.MaxStringLen { return nil, tengo.ErrStringLimit } diff --git a/vm.go b/vm.go index 373a6b11..9fb3e881 100644 --- a/vm.go +++ b/vm.go @@ -49,6 +49,7 @@ type VM struct { childCtl vmChildCtl In io.Reader Out io.Writer + Args []string } // NewVM creates a VM. @@ -72,6 +73,7 @@ func NewVM( childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, In: os.Stdin, Out: os.Stdout, + Args: os.Args, } v.frames[0].fn = bytecode.MainFunction v.frames[0].ip = -1 @@ -128,6 +130,7 @@ func (v *VM) ShallowClone() *VM { childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, In: v.In, Out: v.Out, + Args: v.Args, } // set to empty entry From a14d22ead7d9493d77d366712f6b299d88a6d0b3 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Thu, 20 May 2021 18:45:47 +0000 Subject: [PATCH 16/20] make times.sleep abortable and nolonger hide ErrVMAborted --- goroutinevm.go | 4 ++-- stdlib/times.go | 26 +++++++++++++++++++++----- vm.go | 13 ++++++------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/goroutinevm.go b/goroutinevm.go index 52826e3d..d2263886 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -233,7 +233,7 @@ func (oc objchan) send(args ...Object) (Object, error) { return nil, ErrWrongNumArguments } select { - case <-vm.abortChan: + case <-vm.AbortChan: return nil, ErrVMAborted case oc <- args[0]: } @@ -249,7 +249,7 @@ func (oc objchan) recv(args ...Object) (Object, error) { return nil, ErrWrongNumArguments } select { - case <-vm.abortChan: + case <-vm.AbortChan: return nil, ErrVMAborted case obj, ok := <-oc: if ok { diff --git a/stdlib/times.go b/stdlib/times.go index 0b6f7bd4..98d1d8d6 100644 --- a/stdlib/times.go +++ b/stdlib/times.go @@ -40,9 +40,10 @@ var timesModule = map[string]tengo.Object{ "october": &tengo.Int{Value: int64(time.October)}, "november": &tengo.Int{Value: int64(time.November)}, "december": &tengo.Int{Value: int64(time.December)}, - "sleep": &tengo.UserFunction{ - Name: "sleep", - Value: timesSleep, + "sleep": &tengo.BuiltinFunction{ + Name: "sleep", + Value: timesSleep, + NeedVMObj: true, }, // sleep(int) "parse_duration": &tengo.UserFunction{ Name: "parse_duration", @@ -183,6 +184,8 @@ var timesModule = map[string]tengo.Object{ } func timesSleep(args ...tengo.Object) (ret tengo.Object, err error) { + vm := args[0].(*tengo.VMObj).Value + args = args[1:] // the first arg is VMObj inserted by VM if len(args) != 1 { err = tengo.ErrWrongNumArguments return @@ -197,10 +200,23 @@ func timesSleep(args ...tengo.Object) (ret tengo.Object, err error) { } return } - - time.Sleep(time.Duration(i1)) ret = tengo.UndefinedValue + if time.Duration(i1) <= time.Second { + time.Sleep(time.Duration(i1)) + return + } + done := make(chan struct{}) + go func() { + time.Sleep(time.Duration(i1)) + done <- struct{}{} + }() + + select { + case <-vm.AbortChan: + return nil, tengo.ErrVMAborted + case <-done: + } return } diff --git a/vm.go b/vm.go index 9fb3e881..be8c2042 100644 --- a/vm.go +++ b/vm.go @@ -45,7 +45,7 @@ type VM struct { maxAllocs int64 allocs int64 err error - abortChan chan struct{} + AbortChan chan struct{} childCtl vmChildCtl In io.Reader Out io.Writer @@ -69,7 +69,7 @@ func NewVM( framesIndex: 1, ip: -1, maxAllocs: maxAllocs, - abortChan: make(chan struct{}), + AbortChan: make(chan struct{}), childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, In: os.Stdin, Out: os.Stdout, @@ -88,7 +88,7 @@ func (v *VM) Abort() { return } atomic.StoreInt64(&v.aborting, 1) - close(v.abortChan) // broadcast to all receivers + close(v.AbortChan) // broadcast to all receivers v.childCtl.Lock() for cvm := range v.childCtl.vmMap { cvm.Abort() @@ -126,7 +126,7 @@ func (v *VM) ShallowClone() *VM { framesIndex: 1, ip: -1, maxAllocs: v.maxAllocs, - abortChan: make(chan struct{}), + AbortChan: make(chan struct{}), childCtl: vmChildCtl{vmMap: make(map[*VM]struct{})}, In: v.In, Out: v.Out, @@ -264,9 +264,8 @@ func (v *VM) callStack(frames []frame) string { func (v *VM) postRun() (err error) { err = v.err - // ErrVMAborted is user behavior thus it is not an actual runtime error - if errors.Is(err, ErrVMAborted) { - err = nil + if err == nil && atomic.LoadInt64(&v.aborting) == 1 { + err = ErrVMAborted // indicate VM was aborted } if err != nil { var e ErrPanic From 367afb15b384c41309d63e938227654cb1461598 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Thu, 27 May 2021 17:54:05 +0000 Subject: [PATCH 17/20] only treat ErrVMAborted in root VM as error --- vm.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vm.go b/vm.go index be8c2042..6ed9ef00 100644 --- a/vm.go +++ b/vm.go @@ -99,6 +99,9 @@ func (v *VM) Abort() { // Run starts the execution. func (v *VM) Run() (err error) { _, err = v.RunCompiled(nil) + if err == nil && atomic.LoadInt64(&v.aborting) == 1 { + err = ErrVMAborted // root VM was aborted + } return } @@ -264,8 +267,9 @@ func (v *VM) callStack(frames []frame) string { func (v *VM) postRun() (err error) { err = v.err - if err == nil && atomic.LoadInt64(&v.aborting) == 1 { - err = ErrVMAborted // indicate VM was aborted + // ErrVMAborted is user behavior thus it is not an actual runtime error + if errors.Is(err, ErrVMAborted) { + err = nil } if err != nil { var e ErrPanic From b979ae1580d4521fb843d3b7d04cb445c5a4f3e1 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Sat, 29 May 2021 04:00:03 +0000 Subject: [PATCH 18/20] fix times.sleep goroutine leak on abort case --- stdlib/times.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stdlib/times.go b/stdlib/times.go index 98d1d8d6..a82ff964 100644 --- a/stdlib/times.go +++ b/stdlib/times.go @@ -209,7 +209,10 @@ func timesSleep(args ...tengo.Object) (ret tengo.Object, err error) { done := make(chan struct{}) go func() { time.Sleep(time.Duration(i1)) - done <- struct{}{} + select { + case <-vm.AbortChan: + case done <- struct{}{}: + } }() select { From 3a929e2c0be8dd1535df80c374cc56ab905f874e Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Mon, 31 May 2021 16:58:26 +0000 Subject: [PATCH 19/20] check aborting status before starting child VM and nil child VM to gc after it is done --- goroutinevm.go | 5 ++++- vm.go | 41 ++++++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/goroutinevm.go b/goroutinevm.go index d2263886..993abb4f 100644 --- a/goroutinevm.go +++ b/goroutinevm.go @@ -69,7 +69,9 @@ func builtinGovm(args ...Object) (Object, error) { callers = vm.callers() } - vm.addChild(gvm.VM) + if err := vm.addChild(gvm.VM); err != nil { + return nil, err + } go func() { var val Object var err error @@ -85,6 +87,7 @@ func builtinGovm(args ...Object) (Object, error) { } gvm.waitChan <- ret{val, err} vm.delChild(gvm.VM) + gvm.VM = nil }() if cfn != nil { diff --git a/vm.go b/vm.go index 6ed9ef00..c01f5fe8 100644 --- a/vm.go +++ b/vm.go @@ -82,22 +82,9 @@ func NewVM( return v } -// Abort aborts the execution of current VM and all its descendant VMs. -func (v *VM) Abort() { - if atomic.LoadInt64(&v.aborting) != 0 { - return - } - atomic.StoreInt64(&v.aborting, 1) - close(v.AbortChan) // broadcast to all receivers - v.childCtl.Lock() - for cvm := range v.childCtl.vmMap { - cvm.Abort() - } - v.childCtl.Unlock() -} - // Run starts the execution. func (v *VM) Run() (err error) { + atomic.StoreInt64(&v.aborting, 0) _, err = v.RunCompiled(nil) if err == nil && atomic.LoadInt64(&v.aborting) == 1 { err = ErrVMAborted // root VM was aborted @@ -186,8 +173,6 @@ func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (val Object, err v.ip = -1 v.allocs = v.maxAllocs + 1 - atomic.StoreInt64(&v.aborting, 0) - defer func() { if perr := recover(); perr != nil { v.err = ErrPanic{perr, debug.Stack()} @@ -223,13 +208,31 @@ func (v *VM) addError(err error) { v.childCtl.Unlock() } -func (v *VM) addChild(cvm *VM) { +// Abort aborts the execution of current VM and all its descendant VMs. +func (v *VM) Abort() { + if atomic.LoadInt64(&v.aborting) != 0 { + return + } + v.childCtl.Lock() + atomic.StoreInt64(&v.aborting, 1) + close(v.AbortChan) // broadcast to all receivers + for cvm := range v.childCtl.vmMap { + cvm.Abort() + } + v.childCtl.Unlock() +} + +func (v *VM) addChild(cvm *VM) error { + v.childCtl.Lock() + defer v.childCtl.Unlock() + if atomic.LoadInt64(&v.aborting) != 0 { + return ErrVMAborted + } v.childCtl.Add(1) if cvm != nil { - v.childCtl.Lock() v.childCtl.vmMap[cvm] = struct{}{} - v.childCtl.Unlock() } + return nil } func (v *VM) delChild(cvm *VM) { From 4cfaeebd522880ab8b5f8ed2123b333c8b3a5cb4 Mon Sep 17 00:00:00 2001 From: Bai Yingjie Date: Mon, 31 May 2021 17:08:27 +0000 Subject: [PATCH 20/20] Start from small stack and frame space and grow on demand This reduce memory foot print significantly when there are large number of small VMs running. --- vm.go | 81 ++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/vm.go b/vm.go index c01f5fe8..3557fdbb 100644 --- a/vm.go +++ b/vm.go @@ -32,11 +32,11 @@ type vmChildCtl struct { // VM is a virtual machine that executes the bytecode compiled by Compiler. type VM struct { constants []Object - stack [StackSize]Object + stack []Object sp int globals []Object fileSet *parser.SourceFileSet - frames [MaxFrames]frame + frames []*frame framesIndex int curFrame *frame curInsts []byte @@ -52,6 +52,11 @@ type VM struct { Args []string } +const ( + initialStackSize = 64 + initialFrames = 16 +) + // NewVM creates a VM. func NewVM( bytecode *Bytecode, @@ -66,6 +71,7 @@ func NewVM( sp: 0, globals: globals, fileSet: bytecode.FileSet, + frames: make([]*frame, 0, initialFrames), framesIndex: 1, ip: -1, maxAllocs: maxAllocs, @@ -75,10 +81,11 @@ func NewVM( Out: os.Stdout, Args: os.Args, } - v.frames[0].fn = bytecode.MainFunction - v.frames[0].ip = -1 - v.curFrame = &v.frames[0] - v.curInsts = v.curFrame.fn.Instructions + frame := &frame{ + fn: bytecode.MainFunction, + ip: -1, + } + v.frames = append(v.frames, frame) return v } @@ -108,11 +115,12 @@ var emptyEntry = &CompiledFunction{ // The copy shares the underlying globals, constants with the original. // ShallowClone is typically followed by RunCompiled to run user supplied compiled function. func (v *VM) ShallowClone() *VM { - shallowClone := &VM{ + vClone := &VM{ constants: v.constants, sp: 0, globals: v.globals, fileSet: v.fileSet, + frames: make([]*frame, 0, initialFrames), framesIndex: 1, ip: -1, maxAllocs: v.maxAllocs, @@ -122,14 +130,12 @@ func (v *VM) ShallowClone() *VM { Out: v.Out, Args: v.Args, } - - // set to empty entry - shallowClone.frames[0].fn = emptyEntry - shallowClone.frames[0].ip = -1 - shallowClone.curFrame = &v.frames[0] - shallowClone.curInsts = v.curFrame.fn.Instructions - - return shallowClone + frame := &frame{ + fn: emptyEntry, + ip: -1, + } + vClone.frames = append(vClone.frames, frame) + return vClone } // constract wrapper function func(fn, ...args){ return fn(args...) } @@ -145,8 +151,14 @@ var funcWrapper = &CompiledFunction{ VarArgs: true, } +func (v *VM) releaseSpace() { + v.stack = nil + v.frames = append(make([]*frame, 0, initialFrames), v.frames[0]) +} + // RunCompiled run the VM with user supplied function fn. func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (val Object, err error) { + v.stack = make([]Object, initialStackSize) if fn == nil { // normal Run // reset VM states v.sp = 0 @@ -164,10 +176,9 @@ func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (val Object, err } v.sp = 2 + len(args) v.frames[0].fn = entry - v.frames[0].ip = -1 } - v.curFrame = &v.frames[0] + v.curFrame = v.frames[0] v.curInsts = v.curFrame.fn.Instructions v.framesIndex = 1 v.ip = -1 @@ -179,12 +190,11 @@ func (v *VM) RunCompiled(fn *CompiledFunction, args ...Object) (val Object, err v.Abort() // run time panic should trigger abort chain } v.childCtl.Wait() // waits for all child VMs to exit - if err = v.postRun(); err != nil { - return - } + err = v.postRun() if fn != nil && atomic.LoadInt64(&v.aborting) == 0 { val = v.stack[v.sp-1] } + v.releaseSpace() }() val = UndefinedValue @@ -249,7 +259,7 @@ func (v *VM) callers() (frames []frame) { curFrame.ip = v.ip - 1 frames = append(frames, curFrame) for i := v.framesIndex - 1; i >= 1; i-- { - curFrame = v.frames[i-1] + curFrame = *v.frames[i-1] frames = append(frames, curFrame) } return frames @@ -765,12 +775,14 @@ func (v *VM) run() { v.sp-- switch arr := v.stack[v.sp].(type) { case *Array: + v.checkGrowStack(len(arr.Value)) for _, item := range arr.Value { v.stack[v.sp] = item v.sp++ } numArgs += len(arr.Value) - 1 case *ImmutableArray: + v.checkGrowStack(len(arr.Value)) for _, item := range arr.Value { v.stack[v.sp] = item v.sp++ @@ -834,7 +846,10 @@ func (v *VM) run() { // update call frame v.curFrame.ip = v.ip // store current ip before call - v.curFrame = &(v.frames[v.framesIndex]) + if v.framesIndex >= len(v.frames) { + v.frames = append(v.frames, &frame{}) + } + v.curFrame = v.frames[v.framesIndex] v.curFrame.fn = callee v.curFrame.freeVars = callee.Free v.curFrame.basePointer = v.sp - numArgs @@ -895,7 +910,7 @@ func (v *VM) run() { } //v.sp-- v.framesIndex-- - v.curFrame = &v.frames[v.framesIndex-1] + v.curFrame = v.frames[v.framesIndex-1] v.curInsts = v.curFrame.fn.Instructions v.ip = v.curFrame.ip //v.sp = lastFrame.basePointer - 1 @@ -1091,7 +1106,27 @@ func (v *VM) run() { v.err = fmt.Errorf("unknown opcode: %d", v.curInsts[v.ip]) return } + v.checkGrowStack(0) + } +} + +func (v *VM) checkGrowStack(added int) { + should := v.sp + added + if should < len(v.stack) { + return + } + if should >= StackSize { + v.err = ErrStackOverflow + return + } + roundup := initialStackSize + newSize := len(v.stack) * 2 + if should > newSize { + newSize = (should + roundup) / roundup * roundup } + new := make([]Object, newSize) + copy(new, v.stack) + v.stack = new } // IsStackEmpty tests if the stack is empty or not.