module refactor (#148)

* wip * move print and JSON functions to modules * builtin functions are not replacable now * builtin functions are added for default nil symbol table * importables: builtin modules and source modules * refactoring runtime tests * fix tests * update documentation * cleanup * clean up cli * fix REPL prints
d5 · Mar 18, 2019 · 61890b1 · 61890b1
1 parent 052ae59
commit 61890b1
Show file tree

Hide file tree

Showing 74 changed files with 1,629 additions and 1,729 deletions.
diff --git a/Makefile b/Makefile
@@ -1,10 +1,13 @@
 vet:
 	go vet ./...
 
+generate:
+	go generate ./...
+
 lint:
 	golint -set_exit_status ./...
 
-test: vet lint
+test: generate vet lint
 	go test -race -cover ./...
 
 fmt:

diff --git a/cli/cli.go b/cli/cli.go
@@ -23,33 +23,28 @@ const (
 	replPrompt    = ">> "
 )
 
-//Options represent REPL options
+// Options represent CLI options
 type Options struct {
-	//Compile output file
+	// Compile output file
 	CompileOutput string
 
-	//Show help flag
+	// Show help flag
 	ShowHelp bool
 
-	//Show version flag
+	// Show version flag
 	ShowVersion bool
 
-	//Input file
+	// Input file
 	InputFile string
 
-	//Version
+	// Version
 	Version string
 
-	//Builtin modules
-	BuiltinModules map[string]objects.Object
+	// Import modules
+	Modules map[string]objects.Importable
 }
 
-var (
-	bm             map[string]bool
-	builtinModules map[string]objects.Object
-)
-
-//Run REPL
+// Run CLI
 func Run(options *Options) {
 	if options.ShowHelp {
 		doHelp()
@@ -59,15 +54,9 @@ func Run(options *Options) {
 		return
 	}
 
-	builtinModules = options.BuiltinModules
-	bm = make(map[string]bool, len(builtinModules))
-	for k := range builtinModules {
-		bm[k] = true
-	}
-
 	if options.InputFile == "" {
 		// REPL
-		runREPL(os.Stdin, os.Stdout)
+		runREPL(options.Modules, os.Stdin, os.Stdout)
 		return
 	}
 
@@ -78,12 +67,12 @@ func Run(options *Options) {
 	}
 
 	if options.CompileOutput != "" {
-		if err := compileOnly(inputData, options.InputFile, options.CompileOutput); err != nil {
+		if err := compileOnly(options.Modules, inputData, options.InputFile, options.CompileOutput); err != nil {
 			_, _ = fmt.Fprintln(os.Stderr, err.Error())
 			os.Exit(1)
 		}
 	} else if filepath.Ext(options.InputFile) == sourceFileExt {
-		if err := compileAndRun(inputData, options.InputFile); err != nil {
+		if err := compileAndRun(options.Modules, inputData, options.InputFile); err != nil {
 			_, _ = fmt.Fprintln(os.Stderr, err.Error())
 			os.Exit(1)
 		}
@@ -127,8 +116,8 @@ func doHelp() {
 	fmt.Println()
 }
 
-func compileOnly(data []byte, inputFile, outputFile string) (err error) {
-	bytecode, err := compileSrc(data, filepath.Base(inputFile))
+func compileOnly(modules map[string]objects.Importable, data []byte, inputFile, outputFile string) (err error) {
+	bytecode, err := compileSrc(modules, data, filepath.Base(inputFile))
 	if err != nil {
 		return
 	}
@@ -159,13 +148,13 @@ func compileOnly(data []byte, inputFile, outputFile string) (err error) {
 	return
 }
 
-func compileAndRun(data []byte, inputFile string) (err error) {
-	bytecode, err := compileSrc(data, filepath.Base(inputFile))
+func compileAndRun(modules map[string]objects.Importable, data []byte, inputFile string) (err error) {
+	bytecode, err := compileSrc(modules, data, filepath.Base(inputFile))
 	if err != nil {
 		return
 	}
 
-	machine := runtime.NewVM(bytecode, nil, nil, builtinModules, -1)
+	machine := runtime.NewVM(bytecode, nil, -1)
 
 	err = machine.Run()
 	if err != nil {
@@ -182,7 +171,7 @@ func runCompiled(data []byte) (err error) {
 		return
 	}
 
-	machine := runtime.NewVM(bytecode, nil, nil, builtinModules, -1)
+	machine := runtime.NewVM(bytecode, nil, -1)
 
 	err = machine.Run()
 	if err != nil {
@@ -192,7 +181,7 @@ func runCompiled(data []byte) (err error) {
 	return
 }
 
-func runREPL(in io.Reader, out io.Writer) {
+func runREPL(modules map[string]objects.Importable, in io.Reader, out io.Writer) {
 	stdin := bufio.NewScanner(in)
 
 	fileSet := source.NewFileSet()
@@ -203,6 +192,28 @@ func runREPL(in io.Reader, out io.Writer) {
 		symbolTable.DefineBuiltin(idx, fn.Name)
 	}
 
+	// embed println function
+	symbol := symbolTable.Define("__repl_println__")
+	globals[symbol.Index] = &objects.UserFunction{
+		Name: "println",
+		Value: func(args ...objects.Object) (ret objects.Object, err error) {
+			var printArgs []interface{}
+			for _, arg := range args {
+				if _, isUndefined := arg.(*objects.Undefined); isUndefined {
+					printArgs = append(printArgs, "<undefined>")
+				} else {
+					s, _ := objects.ToString(arg)
+					printArgs = append(printArgs, s)
+				}
+			}
+
+			printArgs = append(printArgs, "\n")
+			_, _ = fmt.Print(printArgs...)
+
+			return
+		},
+	}
+
 	var constants []objects.Object
 
 	for {
@@ -225,15 +236,15 @@ func runREPL(in io.Reader, out io.Writer) {
 
 		file = addPrints(file)
 
-		c := compiler.NewCompiler(srcFile, symbolTable, constants, bm, nil)
+		c := compiler.NewCompiler(srcFile, symbolTable, constants, modules, nil)
 		if err := c.Compile(file); err != nil {
 			_, _ = fmt.Fprintln(out, err.Error())
 			continue
 		}
 
 		bytecode := c.Bytecode()
 
-		machine := runtime.NewVM(bytecode, globals, nil, builtinModules, -1)
+		machine := runtime.NewVM(bytecode, globals, -1)
 		if err := machine.Run(); err != nil {
 			_, _ = fmt.Fprintln(out, err.Error())
 			continue
@@ -243,7 +254,7 @@ func runREPL(in io.Reader, out io.Writer) {
 	}
 }
 
-func compileSrc(src []byte, filename string) (*compiler.Bytecode, error) {
+func compileSrc(modules map[string]objects.Importable, src []byte, filename string) (*compiler.Bytecode, error) {
 	fileSet := source.NewFileSet()
 	srcFile := fileSet.AddFile(filename, -1, len(src))
 
@@ -253,7 +264,9 @@ func compileSrc(src []byte, filename string) (*compiler.Bytecode, error) {
 		return nil, err
 	}
 
-	c := compiler.NewCompiler(srcFile, nil, nil, bm, nil)
+	c := compiler.NewCompiler(srcFile, nil, nil, modules, nil)
+	c.EnableFileImport(true)
+
 	if err := c.Compile(file); err != nil {
 		return nil, err
 	}
@@ -266,14 +279,13 @@ func compileSrc(src []byte, filename string) (*compiler.Bytecode, error) {
 
 func addPrints(file *ast.File) *ast.File {
 	var stmts []ast.Stmt
+
 	for _, s := range file.Stmts {
 		switch s := s.(type) {
 		case *ast.ExprStmt:
 			stmts = append(stmts, &ast.ExprStmt{
 				Expr: &ast.CallExpr{
-					Func: &ast.Ident{
-						Name: "print",
-					},
+					Func: &ast.Ident{Name: "__repl_println__"},
 					Args: []ast.Expr{s.Expr},
 				},
 			})
@@ -284,7 +296,7 @@ func addPrints(file *ast.File) *ast.File {
 			stmts = append(stmts, &ast.ExprStmt{
 				Expr: &ast.CallExpr{
 					Func: &ast.Ident{
-						Name: "print",
+						Name: "__repl_println__",
 					},
 					Args: s.LHS,
 				},

diff --git a/cmd/bench/main.go b/cmd/bench/main.go
@@ -210,7 +210,7 @@ func runVM(bytecode *compiler.Bytecode) (time.Duration, objects.Object, error) {
 
 	start := time.Now()
 
-	v := runtime.NewVM(bytecode, globals, nil, nil, -1)
+	v := runtime.NewVM(bytecode, globals, -1)
 	if err := v.Run(); err != nil {
 		return time.Since(start), nil, err
 	}

diff --git a/cmd/tengo/main.go b/cmd/tengo/main.go
@@ -4,7 +4,6 @@ import (
 	"flag"
 
 	"github.com/d5/tengo/cli"
-	"github.com/d5/tengo/objects"
 	"github.com/d5/tengo/stdlib"
 )
 
@@ -23,17 +22,12 @@ func init() {
 }
 
 func main() {
-	builtinModules := make(map[string]objects.Object, len(stdlib.Modules))
-	for k, mod := range stdlib.Modules {
-		builtinModules[k] = mod
-	}
-
 	cli.Run(&cli.Options{
-		ShowHelp:       showHelp,
-		ShowVersion:    showVersion,
-		Version:        version,
-		CompileOutput:  compileOutput,
-		BuiltinModules: builtinModules,
-		InputFile:      flag.Arg(0),
+		ShowHelp:      showHelp,
+		ShowVersion:   showVersion,
+		Version:       version,
+		CompileOutput: compileOutput,
+		Modules:       stdlib.GetModules(stdlib.AllModuleNames()...),
+		InputFile:     flag.Arg(0),
 	})
 }
diff --git a/compiler/bytecode_optimize.go b/compiler/bytecode_optimize.go
@@ -16,14 +16,25 @@ func (b *Bytecode) RemoveDuplicates() {
 	strings := make(map[string]int)
 	floats := make(map[float64]int)
 	chars := make(map[rune]int)
+	immutableMaps := make(map[string]int) // for modules
 
 	for curIdx, c := range b.Constants {
 		switch c := c.(type) {
 		case *objects.CompiledFunction:
 			// add to deduped list
 			indexMap[curIdx] = len(deduped)
 			deduped = append(deduped, c)
-			continue
+		case *objects.ImmutableMap:
+			modName := c.Value["__module_name__"].(*objects.String).Value
+			newIdx, ok := immutableMaps[modName]
+			if modName != "" && ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				immutableMaps[modName] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
 		case *objects.Int:
 			if newIdx, ok := ints[c.Value]; ok {
 				indexMap[curIdx] = newIdx