Add template support to evaluation details.

internal/engine/errors/errors.go

@@ -20,17 +20,55 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"strings"
+	"text/template"
 
 	"github.com/stacklok/minder/internal/db"
 )
 
+const (
+	maxDetailsMessageSize int64 = 1 << 10
+)
+
 // ErrInternal is an error that occurs when there is an internal error in the minder engine.
 var ErrInternal = errors.New("internal minder error")
 
+type limitedWriter struct {
+	w io.Writer
+	n int64
+}
+
+var _ io.Writer = (*limitedWriter)(nil)
+
+func (l *limitedWriter) Write(p []byte) (int, error) {
+	if l.n < 0 {
+		return 0, io.ErrShortBuffer
+	}
+	if int64(len(p)) > l.n {
+		return 0, io.ErrShortBuffer
+	}
+	n, err := l.w.Write(p)
+	l.n -= int64(n)
+	return n, err
+}
+
+// LimitedWriter returns a writer that allows up to `n` bytes being
+// written. If more than `n` total bytes are written,
+// `io.ErrShortBuffer` is returned.
+func LimitedWriter(w io.Writer, n int64) io.Writer {
+	return &limitedWriter{
+		w: w,
+		n: n,
+	}
+}
+
 // EvaluationError is a custom error type for evaluation errors.
 type EvaluationError struct {
-	Base error
-	Msg  string
+	Base         error
+	Msg          string
+	Template     string
+	TemplateArgs any
 }
 
 // Unwrap returns the base error, allowing errors.Is to work with wrapped errors.
@@ -43,6 +81,42 @@ func (e *EvaluationError) Error() string {
 	return fmt.Sprintf("%v: %s", e.Base, e.Msg)
 }
 
+// Details returns a pretty-printed message detailing the reason of
+// the failure.
+func (e *EvaluationError) Details() string {
+	if e.Template == "" {
+		return e.Msg
+	}
+	tmpl, err := template.New("error").Parse(e.Template)
+	if err != nil {
+		return e.Error()
+	}
+
+	var buf strings.Builder
+	w := LimitedWriter(&buf, maxDetailsMessageSize)
+	if err := tmpl.Execute(w, e.TemplateArgs); err != nil {
+		return e.Error()
+	}
+	return buf.String()
+}
+
+// NewDetailedErrEvaluationFailed creates a new evaluation error with
+// a given error message and a templated detail message.
+func NewDetailedErrEvaluationFailed(
+	tmpl string,
+	tmplArgs any,
+	sfmt string,
+	args ...any,
+) error {
+	formatted := fmt.Sprintf(sfmt, args...)
+	return &EvaluationError{
+		Base:         ErrEvaluationFailed,
+		Msg:          formatted,
+		Template:     tmpl,
+		TemplateArgs: tmplArgs,
+	}
+}
+
 // ErrEvaluationFailed is an error that occurs during evaluation of a rule.
 var ErrEvaluationFailed = errors.New("evaluation failure")
 
@@ -131,12 +205,15 @@ func ErrorAsEvalStatus(err error) db.EvalStatusTypes {
 // ErrorAsEvalDetails returns the evaluation details for a given error
 func ErrorAsEvalDetails(err error) string {
 	var evalErr *EvaluationError
+	if errors.As(err, &evalErr) && evalErr.Template != "" {
+		return evalErr.Details()
+	}
 	if errors.As(err, &evalErr) {
 		return evalErr.Msg
-	} else if err != nil {
+	}
+	if err != nil {
 		return err.Error()
 	}
-
 	return ""
 }
 

internal/engine/errors/errors_test.go

@@ -0,0 +1,147 @@
+// Copyright 2024 Stacklok, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package errors
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/stacklok/minder/internal/engine/eval/templates"
+)
+
+func TestLegacyEvaluationDetailRendering(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		msg     string
+		args    []any
+		error   string
+		details string
+	}{
+		{
+			name:    "legacy",
+			msg:     "format: %s",
+			args:    []any{"this is the message"},
+			error:   "evaluation failure: format: this is the message",
+			details: "format: this is the message",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			err := NewErrEvaluationFailed(
+				tt.msg,
+				tt.args...,
+			)
+
+			require.Equal(t, tt.error, err.Error())
+			evalErr, ok := err.(*EvaluationError)
+			require.True(t, ok)
+			require.Equal(t, tt.details, evalErr.Msg)
+		})
+	}
+}
+
+func TestEvaluationDetailRendering(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		msg     string
+		msgArgs []any
+		tmpl    string
+		args    any
+		error   string
+		details string
+	}{
+		{
+			name:    "legacy",
+			msg:     "this is the message",
+			tmpl:    "",
+			args:    nil,
+			error:   "evaluation failure: this is the message",
+			details: "this is the message",
+		},
+		{
+			name:    "empty template",
+			msg:     "this is the message",
+			tmpl:    "",
+			args:    nil,
+			error:   "evaluation failure: this is the message",
+			details: "this is the message",
+		},
+		{
+			name:    "simple template",
+			msg:     "this is the message",
+			tmpl:    "fancy template with {{ . }}",
+			args:    "fancy message",
+			error:   "evaluation failure: this is the message",
+			details: "fancy template with fancy message",
+		},
+		{
+			name:    "complex template",
+			msg:     "this is the message",
+			tmpl:    "fancy template with {{ range $idx, $val := . }}{{ if $idx }}, {{ end }}{{ . }}{{ end }}",
+			args:    []any{"many", "many", "many messages"},
+			error:   "evaluation failure: this is the message",
+			details: "fancy template with many, many, many messages",
+		},
+		{
+			name:    "enforced limit",
+			msg:     "this is the message",
+			tmpl:    "fancy template with {{ . }}",
+			args:    strings.Repeat("A", 1025),
+			error:   "evaluation failure: this is the message",
+			details: "evaluation failure: this is the message",
+		},
+		// vulncheck template
+		{
+			name:    "vulncheck template",
+			msg:     "this is the message",
+			tmpl:    templates.VulncheckTemplate,
+			args:    map[string]any{"packages": []string{"boto3", "urllib3", "python-oauth2"}},
+			error:   "evaluation failure: this is the message",
+			details: "Vulnerable packages found:\n* `boto3`\n* `urllib3`\n* `python-oauth2`\n",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			err := NewDetailedErrEvaluationFailed(
+				tt.tmpl,
+				tt.args,
+				tt.msg,
+				tt.msgArgs...,
+			)
+
+			require.Equal(t, tt.error, err.Error())
+			evalErr, ok := err.(*EvaluationError)
+			require.True(t, ok)
+			require.Equal(t, tt.details, evalErr.Details())
+			require.LessOrEqual(t, len(evalErr.Details()), 1024)
+		})
+	}
+}

internal/engine/eval/templates/templates.go

@@ -0,0 +1,27 @@
+// Copyright 2024 Stacklok, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package templates contains template strings for evaluation details.
+package templates
+
+import (
+	// This comment makes the linter happy.
+	_ "embed"
+)
+
+// VulncheckTemplate is the template for evaluation details of the
+// `vulncheck` evaluation engine.
+//
+//go:embed vulncheckTemplate.tmpl
+var VulncheckTemplate string

internal/engine/eval/templates/vulncheckTemplate.tmpl

@@ -0,0 +1,4 @@
+Vulnerable packages found:
+{{- range .packages }}
+* `{{- . -}}`
+{{- end }}

internal/engine/eval/vulncheck/vulncheck.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rs/zerolog"
 
 	evalerrors "github.com/stacklok/minder/internal/engine/errors"
+	"github.com/stacklok/minder/internal/engine/eval/templates"
 	engif "github.com/stacklok/minder/internal/engine/interfaces"
 	pbinternal "github.com/stacklok/minder/internal/proto"
 	provifv1 "github.com/stacklok/minder/pkg/providers/v1"
@@ -59,7 +60,12 @@ func (e *Evaluator) Eval(ctx context.Context, pol map[string]any, res *engif.Res
 	}
 
 	if len(vulnerablePackages) > 0 {
-		return evalerrors.NewErrEvaluationFailed("vulnerable packages: %s", strings.Join(vulnerablePackages, ","))
+		return evalerrors.NewDetailedErrEvaluationFailed(
+			templates.VulncheckTemplate,
+			vulnerablePackages,
+			"vulnerable packages: %s",
+			strings.Join(vulnerablePackages, ","),
+		)
 	}
 
 	return nil

internal/engine/eval/vulncheck/vulncheck_test.go

@@ -0,0 +1,68 @@
+// Copyright 2024 Stacklok, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vulncheck
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	evalerrors "github.com/stacklok/minder/internal/engine/errors"
+	"github.com/stacklok/minder/internal/engine/eval/templates"
+)
+
+func TestEvaluationDetailRendering(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		msg     string
+		msgArgs []any
+		tmpl    string
+		args    any
+		error   string
+		details string
+	}{
+		// vulncheck template
+		{
+			name:    "vulncheck template",
+			msg:     "this is the message",
+			tmpl:    templates.VulncheckTemplate,
+			args:    map[string]any{"packages": []string{"boto3", "urllib3", "python-oauth2"}},
+			error:   "evaluation failure: this is the message",
+			details: "Vulnerable packages found:\n* `boto3`\n* `urllib3`\n* `python-oauth2`\n",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			err := evalerrors.NewDetailedErrEvaluationFailed(
+				tt.tmpl,
+				tt.args,
+				tt.msg,
+				tt.msgArgs...,
+			)
+
+			require.Equal(t, tt.error, err.Error())
+			evalErr, ok := err.(*evalerrors.EvaluationError)
+			require.True(t, ok)
+			require.Equal(t, tt.details, evalErr.Details())
+		})
+	}
+}