Introduce Flowpilot - integration

.github/workflows/build-frontend.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: v18.14.2
+          node-version: v20.16.0
 
       - name: Install dependencies
         working-directory: ./frontend

.github/workflows/generate-config-docs.yml

@@ -0,0 +1,56 @@
+name: Generate config reference markdown
+
+on:
+  workflow_dispatch:
+
+jobs:
+  config:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.20'
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.16.0'
+          registry-url: https://registry.npmjs.org/
+
+      - name: Checkout backend
+        uses: actions/checkout@v4
+        with:
+          path: hanko
+
+      - name: Checkout backend wiki
+        uses: actions/checkout@v4
+        with:
+          repository: ${{github.repository}}.wiki
+          path: wiki
+
+      - name: Generate config docs
+        working-directory: ./hanko/backend
+        run: |
+          go generate ./...
+          go run main.go schema json2md
+
+      - name: Clean md file endings
+        working-directory: ./hanko/backend
+        run: |
+          find ./docs/.generated/config/md -type f -name "*.md" -exec sed -i "s/\.md//g" "{}" \;
+
+      - name: Copy generated files
+        working-directory: ./hanko/backend
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/wiki/reference/config
+          rm $GITHUB_WORKSPACE/wiki/reference/config/*.md 2>/dev/null || true
+          cp ./docs/.generated/config/md/*.md $GITHUB_WORKSPACE/wiki/reference/config
+
+      - name: Commit and push to wiki
+        working-directory: ./wiki
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add .
+          git commit -m "action: Autogenerate config reference docs"
+          git push origin HEAD

.gitignore

@@ -14,6 +14,9 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# Generated files
+.generated
+
 # MacOS
 .DS_Store
 

backend/Dockerfile

@@ -30,6 +30,8 @@ COPY template template/
 COPY utils utils/
 COPY mapper mapper/
 COPY webhooks webhooks/
+COPY flow_api flow_api/
+COPY flowpilot flowpilot/
 
 # Build
 RUN go generate ./...

backend/Dockerfile.debug

@@ -27,6 +27,8 @@ COPY rate_limiter rate_limiter/
 COPY thirdparty thirdparty/
 COPY build_info build_info/
 COPY middleware middleware/
+COPY flow_api flow_api/
+COPY flowpilot flowpilot/
 COPY template template/
 COPY utils utils/
 COPY mapper mapper/

backend/audit_log/logger.go

@@ -1,30 +1,30 @@
 package auditlog
 
 import (
-	"fmt"
 	"github.com/gobuffalo/pop/v6"
-	"github.com/gofrs/uuid"
 	"github.com/labstack/echo/v4"
 	zeroLog "github.com/rs/zerolog"
 	zeroLogger "github.com/rs/zerolog/log"
 	"github.com/teamhanko/hanko/backend/config"
 	"github.com/teamhanko/hanko/backend/persistence"
 	"github.com/teamhanko/hanko/backend/persistence/models"
+	"github.com/teamhanko/hanko/backend/utils"
 	"os"
 	"strconv"
 	"time"
 )
 
 type Logger interface {
-	Create(echo.Context, models.AuditLogType, *models.User, error) error
-	CreateWithConnection(*pop.Connection, echo.Context, models.AuditLogType, *models.User, error) error
+	Create(echo.Context, models.AuditLogType, *models.User, error, ...DetailOption) error
+	CreateWithConnection(*pop.Connection, echo.Context, models.AuditLogType, *models.User, error, ...DetailOption) error
 }
 
 type logger struct {
 	persister             persistence.Persister
 	storageEnabled        bool
 	logger                zeroLog.Logger
 	consoleLoggingEnabled bool
+	mustMask              bool
 }
 
 func NewLogger(persister persistence.Persister, cfg config.AuditLog) Logger {
@@ -43,78 +43,108 @@ func NewLogger(persister persistence.Persister, cfg config.AuditLog) Logger {
 		storageEnabled:        cfg.Storage.Enabled,
 		logger:                zeroLog.New(loggerOutput),
 		consoleLoggingEnabled: cfg.ConsoleOutput.Enabled,
+		mustMask:              cfg.Mask,
 	}
 }
 
-func (l *logger) Create(context echo.Context, auditLogType models.AuditLogType, user *models.User, logError error) error {
-	return l.CreateWithConnection(l.persister.GetConnection(), context, auditLogType, user, logError)
-}
+type DetailOption func(map[string]interface{})
 
-func (l *logger) CreateWithConnection(tx *pop.Connection, context echo.Context, auditLogType models.AuditLogType, user *models.User, logError error) error {
-	if l.storageEnabled {
-		err := l.store(tx, context, auditLogType, user, logError)
-		if err != nil {
-			return err
+func Detail(key string, value interface{}) DetailOption {
+	return func(d map[string]interface{}) {
+		if value != "" || value != nil {
+			d[key] = value
 		}
 	}
+}
 
-	if l.consoleLoggingEnabled {
-		l.logToConsole(context, auditLogType, user, logError)
-	}
-
-	return nil
+func (l *logger) Create(context echo.Context, auditLogType models.AuditLogType, user *models.User, logError error, detailOpts ...DetailOption) error {
+	return l.CreateWithConnection(l.persister.GetConnection(), context, auditLogType, user, logError, detailOpts...)
 }
 
-func (l *logger) store(tx *pop.Connection, context echo.Context, auditLogType models.AuditLogType, user *models.User, logError error) error {
-	id, err := uuid.NewV4()
+func (l *logger) CreateWithConnection(tx *pop.Connection, context echo.Context, auditLogType models.AuditLogType, user *models.User, logError error, detailOpts ...DetailOption) error {
+	details := make(map[string]interface{})
+	for _, detailOpt := range detailOpts {
+		detailOpt(details)
+	}
+
+	auditLog, err := models.NewAuditLog(auditLogType, l.getRequestMeta(context), details, user, logError)
 	if err != nil {
-		return fmt.Errorf("failed to create id: %w", err)
+		return err
 	}
 
-	al := models.AuditLog{
-		ID:                id,
-		Type:              auditLogType,
-		Error:             nil,
-		MetaHttpRequestId: context.Response().Header().Get(echo.HeaderXRequestID),
-		MetaUserAgent:     context.Request().UserAgent(),
-		MetaSourceIp:      context.RealIP(),
-		ActorUserId:       nil,
-		ActorEmail:        nil,
+	if l.mustMask {
+		auditLog = l.mask(auditLog)
 	}
 
-	if user != nil {
-		al.ActorUserId = &user.ID
-		if e := user.Emails.GetPrimary(); e != nil {
-			al.ActorEmail = &e.Address
+	if l.storageEnabled {
+		err = l.store(tx, auditLog)
+		if err != nil {
+			return err
 		}
 	}
-	if logError != nil {
-		// check if error is not nil, because else the string (formatted with fmt.Sprintf) would not be empty but look like this: `%!s(<nil>)`
-		tmp := fmt.Sprintf("%s", logError)
-		al.Error = &tmp
+
+	if l.consoleLoggingEnabled {
+		l.logToConsole(auditLog)
 	}
 
-	return l.persister.GetAuditLogPersisterWithConnection(tx).Create(al)
+	return nil
 }
 
-func (l *logger) logToConsole(context echo.Context, auditLogType models.AuditLogType, user *models.User, logError error) {
+func (l *logger) store(tx *pop.Connection, auditLog models.AuditLog) error {
+	return l.persister.GetAuditLogPersisterWithConnection(tx).Create(auditLog)
+}
+
+func (l *logger) logToConsole(auditLog models.AuditLog) {
+	var err string
+	if auditLog.Error != nil {
+		err = *auditLog.Error
+	}
+
 	now := time.Now()
 	loggerEvent := zeroLogger.Log().
 		Str("audience", "audit").
-		Str("type", string(auditLogType)).
-		AnErr("error", logError).
-		Str("http_request_id", context.Response().Header().Get(echo.HeaderXRequestID)).
-		Str("source_ip", context.RealIP()).
-		Str("user_agent", context.Request().UserAgent()).
+		Str("type", string(auditLog.Type)).
+		Str("error", err).
+		Str("http_request_id", auditLog.MetaHttpRequestId).
+		Str("source_ip", auditLog.MetaSourceIp).
+		Str("user_agent", auditLog.MetaUserAgent).
+		Any("details", auditLog.Details).
 		Str("time", now.Format(time.RFC3339Nano)).
 		Str("time_unix", strconv.FormatInt(now.Unix(), 10))
 
-	if user != nil {
-		loggerEvent.Str("user_id", user.ID.String())
-		if e := user.Emails.GetPrimary(); e != nil {
-			loggerEvent.Str("user_email", e.Address)
+	if auditLog.ActorUserId != nil {
+		loggerEvent.Str("user_id", auditLog.ActorUserId.String())
+		if auditLog.ActorEmail != nil {
+			loggerEvent.Str("user_email", *auditLog.ActorEmail)
 		}
 	}
 
 	loggerEvent.Send()
 }
+
+func (l *logger) getRequestMeta(c echo.Context) models.RequestMeta {
+	return models.RequestMeta{
+		HttpRequestId: c.Response().Header().Get(echo.HeaderXRequestID),
+		UserAgent:     c.Request().UserAgent(),
+		SourceIp:      c.RealIP(),
+	}
+}
+
+func (l *logger) mask(auditLog models.AuditLog) models.AuditLog {
+	if auditLog.ActorEmail != nil && *auditLog.ActorEmail != "" {
+		email := utils.MaskEmail(*auditLog.ActorEmail)
+		auditLog.ActorEmail = &email
+	}
+
+	for key, value := range auditLog.Details {
+		if key == "username" {
+			auditLog.Details[key] = utils.MaskUsername(value.(string))
+		}
+
+		if key == "email" {
+			auditLog.Details[key] = utils.MaskEmail(value.(string))
+		}
+	}
+
+	return auditLog
+}

backend/cmd/root.go

@@ -9,6 +9,7 @@ import (
 	"github.com/teamhanko/hanko/backend/cmd/jwk"
 	"github.com/teamhanko/hanko/backend/cmd/jwt"
 	"github.com/teamhanko/hanko/backend/cmd/migrate"
+	"github.com/teamhanko/hanko/backend/cmd/schema"
 	"github.com/teamhanko/hanko/backend/cmd/serve"
 	"github.com/teamhanko/hanko/backend/cmd/siwa"
 	"github.com/teamhanko/hanko/backend/cmd/user"
@@ -29,6 +30,7 @@ func NewRootCmd() *cobra.Command {
 	version.RegisterCommands(cmd)
 	user.RegisterCommands(cmd)
 	siwa.RegisterCommands(cmd)
+	schema.RegisterCommands(cmd)
 
 	return cmd
 }

backend/cmd/schema/json2md.go

@@ -0,0 +1,74 @@
+package schema
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/invopop/jsonschema"
+	"github.com/spf13/cobra"
+	"github.com/teamhanko/hanko/backend/config"
+	"log"
+	"os"
+	"os/exec"
+)
+
+func NewJson2MdCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "json2md",
+		Short: "Generate markdown from JSONSchema",
+		Run: func(cmd *cobra.Command, args []string) {
+			r := new(jsonschema.Reflector)
+			r.DoNotReference = true
+			if err := r.AddGoComments("github.com/teamhanko/hanko/backend", "./config"); err != nil {
+				log.Fatal(err)
+			}
+
+			if err := r.AddGoComments("github.com/teamhanko/hanko/backend", "./ee"); err != nil {
+				log.Fatal(err)
+			}
+
+			s := r.Reflect(&config.Config{})
+			s.Title = "Config"
+
+			data, err := json.MarshalIndent(s, "", "  ")
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			outPath := "./docs/.generated/config"
+			if _, err := os.Stat(outPath); errors.Is(err, os.ErrNotExist) {
+				err := os.MkdirAll(outPath, 0750)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+
+			err = os.WriteFile(fmt.Sprintf("%s/hanko.config.json", outPath), data, 0600)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			out, err := exec.Command("npx",
+				"@adobe/jsonschema2md",
+				"--input=docs/.generated/config",
+				"--out=docs/.generated/config/md",
+				"--schema-extension=config.json",
+				"--example-format=yaml",
+				"--header=false",
+				"--skip=definedinfact",
+				"--skip=typesection",
+				"--schema-out=-",
+				"--properties=format",
+				"--no-readme=true").
+				CombinedOutput()
+
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			fmt.Println(string(out))
+		},
+	}
+
+	return cmd
+}

backend/cmd/schema/root.go

@@ -0,0 +1,17 @@
+package schema
+
+import "github.com/spf13/cobra"
+
+func NewSchemaCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "schema",
+		Short: "JSONSchema related commands",
+		Long:  ``,
+	}
+}
+
+func RegisterCommands(parent *cobra.Command) {
+	cmd := NewSchemaCommand()
+	parent.AddCommand(cmd)
+	cmd.AddCommand(NewJson2MdCommand())
+}