Feature/testing

cmd/falco/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"math"
 	"os"
 	"strings"
 
@@ -12,8 +13,11 @@ import (
 	"github.com/mattn/go-colorable"
 	"github.com/pkg/errors"
 	"github.com/ysugimoto/falco/config"
+	ife "github.com/ysugimoto/falco/interpreter/function/errors"
+	"github.com/ysugimoto/falco/lexer"
 	"github.com/ysugimoto/falco/resolver"
 	"github.com/ysugimoto/falco/terraform"
+	"github.com/ysugimoto/falco/token"
 )
 
 var version string = ""
@@ -27,6 +31,12 @@ var (
 	cyan    = color.New(color.FgCyan)
 	magenta = color.New(color.FgMagenta)
 
+	// Displaying test result needs adding background colors
+	noTestColor = color.New(color.FgBlack, color.BgWhite, color.Bold)
+	passColor   = color.New(color.FgWhite, color.BgGreen, color.Bold)
+	failColor   = color.New(color.FgWhite, color.BgRed, color.Bold)
+	redBold     = color.New(color.FgRed, color.Bold)
+
 	ErrExit = errors.New("exit")
 )
 
@@ -35,6 +45,7 @@ const (
 	subcommandTerraform = "terraform"
 	subcommandLocal     = "local"
 	subcommandStats     = "stats"
+	subcommandTest      = "test"
 )
 
 func write(c *color.Color, format string, args ...interface{}) {
@@ -63,6 +74,7 @@ Subcommands:
     lint      : Run lint (default)
     stats     : Analyze VCL statistics
     local     : Run local simulate server with provided VCLs
+    test      : Run local testing for provided VCLs
 
 Flags:
     -I, --include_path : Add include path
@@ -85,6 +97,9 @@ Get statistics example:
 Local server with debugger example:
 	falco -I . local -debug /path/to/vcl/main.vcl
 
+Local testing with builtin interpreter:
+	falco -I . -I ./tests test /path/to/vcl/main.vcl
+
 Linting with terraform:
     terraform plan -out planned.out
     terraform show -json planned.out | falco -vv terraform
@@ -117,8 +132,8 @@ func main() {
 			resolvers = resolver.NewTerraformResolver(fastlyServices)
 			fetcher = terraform.NewTerraformFetcher(fastlyServices)
 		}
-	case subcommandLocal, subcommandLint, subcommandStats:
-		// "lint", "local" and "stats" command provides single file of service,
+	case subcommandLocal, subcommandLint, subcommandStats, subcommandTest:
+		// "lint", "local", "stats" and "test" command provides single file of service,
 		// then resolvers size is always 1
 		resolvers, err = resolver.NewFileResolvers(c.Commands.At(1), c.IncludePaths)
 	default:
@@ -141,12 +156,10 @@ func main() {
 
 		var exitErr error
 		switch c.Commands.At(0) {
+		case subcommandTest:
+			exitErr = runTest(runner, v)
 		case subcommandLocal:
-			if c.Debug {
-				exitErr = runDebugger(runner, v)
-			} else {
-				exitErr = runSimulator(runner, v)
-			}
+			exitErr = runSimulate(runner, v)
 		case subcommandStats:
 			exitErr = runStats(runner, v)
 		default:
@@ -223,17 +236,9 @@ func runLint(runner *Runner, rslv resolver.Resolver) error {
 	return nil
 }
 
-func runDebugger(runner *Runner, rslv resolver.Resolver) error {
-	if err := runner.Debugger(rslv); err != nil {
-		writeln(red, "Failed to start debugger console: %s", err.Error())
-		return ErrExit
-	}
-	return nil
-}
-
-func runSimulator(runner *Runner, rslv resolver.Resolver) error {
-	if err := runner.Simulator(rslv); err != nil {
-		writeln(red, "Failed to start server: %s", err.Error())
+func runSimulate(runner *Runner, rslv resolver.Resolver) error {
+	if err := runner.Simulate(rslv); err != nil {
+		writeln(red, "Failed to start local simulator: %s", err.Error())
 		return ErrExit
 	}
 	return nil
@@ -257,6 +262,9 @@ func runStats(runner *Runner, rslv resolver.Resolver) error {
 		}
 		return ErrExit
 	}
+	printStats := func(format string, args ...interface{}) {
+		fmt.Fprintf(os.Stdout, format+"\n", args...)
+	}
 
 	printStats(strings.Repeat("=", 80))
 	printStats("| %-76s |", "falco VCL statistics ")
@@ -280,6 +288,76 @@ func runStats(runner *Runner, rslv resolver.Resolver) error {
 	return nil
 }
 
-func printStats(format string, args ...interface{}) {
-	fmt.Fprintf(os.Stdout, format+"\n", args...)
+func runTest(runner *Runner, rslv resolver.Resolver) error {
+	write(white, "Running tests...")
+	results, counter, err := runner.Test(rslv)
+	if err != nil {
+		writeln(red, " Failed.")
+		writeln(red, "Failed to run test: %s", err.Error())
+		return ErrExit
+	}
+
+	// shrthand indent making
+	indent := func(level int) string {
+		return strings.Repeat(" ", level*2)
+	}
+	// print problem line
+	printCodeLine := func(lx *lexer.Lexer, tok token.Token) {
+		problemLine := tok.Line
+		lineFormat := fmt.Sprintf(" %%%dd", int(math.Floor(math.Log10(float64(problemLine+1))+1)))
+		for l := problemLine - 1; l <= problemLine+1; l++ {
+			line, ok := lx.GetLine(l)
+			if !ok {
+				continue
+			}
+			color := white
+			if l == problemLine {
+				color = yellow
+			}
+			writeln(color, "%s "+lineFormat+"| %s", indent(1), l, strings.ReplaceAll(line, "\t", "    "))
+		}
+	}
+
+	writeln(white, " Done.")
+	for _, r := range results {
+		switch {
+		case len(r.Cases) == 0:
+			write(noTestColor, " NO TESTS ")
+			writeln(white, " "+r.Filename)
+		case r.IsPassed():
+			write(passColor, " PASS ")
+			writeln(white, " "+r.Filename)
+		default:
+			write(failColor, " FAIL ")
+			writeln(white, " "+r.Filename)
+		}
+
+		for _, c := range r.Cases {
+			if c.Error != nil {
+				writeln(redBold, "%s●  [%s] %s\n", indent(1), c.Scope, c.Name)
+				writeln(red, "%s%s", indent(2), c.Error.Error())
+				switch e := c.Error.(type) {
+				case *ife.AssertionError:
+					write(white, "%sActual Value: ", indent(2))
+					writeln(red, "%s\n", e.Actual.String())
+					printCodeLine(r.Lexer, e.Token)
+				case *ife.TestingError:
+					writeln(white, "")
+					printCodeLine(r.Lexer, e.Token)
+				}
+				writeln(white, "")
+			} else {
+				writeln(green, "%s✓ [%s] %s", indent(1), c.Scope, c.Name)
+			}
+		}
+	}
+
+	write(green, "%d passed, ", counter.Passes)
+	if len(counter.Fails) > 0 {
+		write(red, "%d failed, ", len(counter.Fails))
+	}
+	write(white, "%d total, ", len(results))
+	writeln(white, "%d assertions", counter.Asserts)
+
+	return nil
 }

cmd/falco/runner.go

@@ -22,6 +22,7 @@ import (
 	"github.com/ysugimoto/falco/plugin"
 	"github.com/ysugimoto/falco/remote"
 	"github.com/ysugimoto/falco/resolver"
+	"github.com/ysugimoto/falco/tester"
 	"github.com/ysugimoto/falco/types"
 )
 
@@ -442,47 +443,56 @@ func (r *Runner) Stats(rslv resolver.Resolver) (*StatsResult, error) {
 	return stats, nil
 }
 
-func (r *Runner) Simulator(rslv resolver.Resolver) error {
+func (r *Runner) Simulate(rslv resolver.Resolver) error {
+	local := r.config.Local
 	options := []icontext.Option{
 		icontext.WithResolver(rslv),
-		icontext.WithMaxBackends(r.config.OverrideMaxBackends),
-		icontext.WithMaxAcls(r.config.OverrideMaxAcls),
+		icontext.WithMaxBackends(local.OverrideMaxBackends),
+		icontext.WithMaxAcls(local.OverrideMaxAcls),
 	}
 	if r.snippets != nil {
 		options = append(options, icontext.WithFastlySnippets(r.snippets))
 	}
 	if v := os.Getenv("FALCO_DEBUG"); v != "" {
 		options = append(options, icontext.WithDebug())
 	}
-	if r.config.OverrideRequest != nil {
-		options = append(options, icontext.WithRequest(r.config.OverrideRequest))
+	if local.OverrideRequest != nil {
+		options = append(options, icontext.WithRequest(local.OverrideRequest))
 	}
 
 	i := interpreter.New(options...)
+
+	// If debugger flag is on, run debugger mode
+	if local.Debug {
+		return debugger.New(interpreter.New(options...)).Run(local.Port)
+	}
+
+	// Otherwise, simply start simulator server
 	mux := http.NewServeMux()
 	mux.Handle("/", i)
 
 	s := &http.Server{
 		Handler: mux,
-		Addr:    ":3124",
+		Addr:    fmt.Sprintf(":%d", local.Port),
 	}
 	writeln(green, "Simulator server starts on 0.0.0.0:3124")
 	return s.ListenAndServe()
 }
 
-func (r *Runner) Debugger(rslv resolver.Resolver) error {
+func (r *Runner) Test(rslv resolver.Resolver) ([]*tester.TestResult, *tester.TestCounter, error) {
+	testing := r.config.Testing
 	options := []icontext.Option{
 		icontext.WithResolver(rslv),
-		icontext.WithMaxBackends(r.config.OverrideMaxBackends),
-		icontext.WithMaxAcls(r.config.OverrideMaxAcls),
+		icontext.WithMaxBackends(testing.OverrideMaxBackends),
+		icontext.WithMaxAcls(testing.OverrideMaxAcls),
 	}
 	if r.snippets != nil {
 		options = append(options, icontext.WithFastlySnippets(r.snippets))
 	}
-	if r.config.OverrideRequest != nil {
-		options = append(options, icontext.WithRequest(r.config.OverrideRequest))
+	if r.config.Testing.OverrideRequest != nil {
+		options = append(options, icontext.WithRequest(testing.OverrideRequest))
 	}
 
-	d := debugger.New(interpreter.New(options...))
-	return d.Run(r.config.Port)
+	i := interpreter.New(options...)
+	return tester.New(testing, i).Run(r.config.Commands.At(1))
 }

config/config.go

@@ -8,10 +8,37 @@ import (
 	"github.com/ysugimoto/twist"
 )
 
-const (
-	configurationFile = ".falco.yaml"
+var (
+	configurationFiles = []string{".falco.yaml", ".falco.yml"}
 )
 
+// Local simulation configuration
+type LocalConfig struct {
+	Port         int      `cli:"p,port" yaml:"port" default:"3124"`
+	Debug        bool     `cli:"debug"`
+	IncludePaths []string // may copied from root field
+
+	// Override resource limits
+	OverrideMaxBackends int `cli:"max_backend" yaml:"override_max_backends"`
+	OverrideMaxAcls     int `cli:"mac_acl" yaml:"override_max_acls"`
+
+	// Override Request configuration
+	OverrideRequest *RequestConfig
+}
+
+// Testing configuration
+type TestConfig struct {
+	Timeout      int      `cli:"timeout" yaml:"timeout"`
+	IncludePaths []string // may copied from root field
+
+	// Override resource limits
+	OverrideMaxBackends int `cli:"max_backend" yaml:"override_max_backends"`
+	OverrideMaxAcls     int `cli:"mac_acl" yaml:"override_max_acls"`
+
+	// Override Request configuration
+	OverrideRequest *RequestConfig
+}
+
 type Config struct {
 	// Root configurations
 	IncludePaths   []string `cli:"I,include_path" yaml:"include_paths"`
@@ -23,16 +50,7 @@ type Config struct {
 	Version        bool     `cli:"V"`
 	Remote         bool     `cli:"r,remote" yaml:"remote"`
 	Json           bool     `cli:"json"`
-	Port           int      `cli:"p,port" yaml:"port" default:"3124"`
 	Request        string   `cli:"request"`
-	Debug          bool     `cli:"debug"`
-
-	// Override resource limits
-	OverrideMaxBackends int `cli:"max_backend" yaml:"override_max_backends"`
-	OverrideMaxAcls     int `cli:"mac_acl" yaml:"override_max_acls"`
-
-	// Override Request configuration
-	OverrideRequest *RequestConfig
 
 	// Remote options, only provided via environment variable
 	FastlyServiceID string `env:"FASTLY_SERVICE_ID"`
@@ -43,6 +61,10 @@ type Config struct {
 
 	// CLI subcommands
 	Commands Commands
+
+	// Local simulator configuration
+	Local   *LocalConfig `yaml:"local"`
+	Testing *TestConfig  `yaml:"testing"`
 }
 
 // Adding type alias in order to implement some methods
@@ -60,7 +82,12 @@ func New(args []string) (*Config, error) {
 	options = append(options, twist.WithEnv(), twist.WithCli(args))
 
 	c := &Config{
-		OverrideRequest: &RequestConfig{},
+		Local: &LocalConfig{
+			OverrideRequest: &RequestConfig{},
+		},
+		Testing: &TestConfig{
+			OverrideRequest: &RequestConfig{},
+		},
 	}
 	if err := twist.Mix(c, options...); err != nil {
 		return nil, errors.WithStack(err)
@@ -75,13 +102,18 @@ func New(args []string) (*Config, error) {
 		c.VerboseInfo = true
 	}
 
-	// Load request configuration
+	// Load request configuration if provided
 	if c.Request != "" {
 		if rc, err := LoadRequestConfig(c.Request); err == nil {
-			c.OverrideRequest = rc
+			c.Local.OverrideRequest = rc
+			c.Testing.OverrideRequest = rc
 		}
 	}
 
+	// Copy common fields
+	c.Local.IncludePaths = c.IncludePaths
+	c.Testing.IncludePaths = c.IncludePaths
+
 	return c, nil
 }
 
@@ -93,9 +125,11 @@ func findConfigFile() (string, error) {
 	}
 
 	for {
-		file := filepath.Join(cwd, configurationFile)
-		if _, err := os.Stat(file); err == nil {
-			return file, nil
+		for _, f := range configurationFiles {
+			file := filepath.Join(cwd, f)
+			if _, err := os.Stat(file); err == nil {
+				return file, nil
+			}
 		}
 
 		cwd = filepath.Dir(cwd)