feat: use river + newman, remove marionette + posthog (#87)

* feat: use river + newan, remove marionette, posthog, sendgrid

Signed-off-by: Sarah Funkhouser <[email protected]>

* fix db, remove exec query feature

Signed-off-by: Sarah Funkhouser <[email protected]>

---------

Signed-off-by: Sarah Funkhouser <[email protected]>

.gitignore

@@ -25,10 +25,10 @@ server.crt
 server.key
 private_key.pem
 public_key.pem
-# Sendgrid test email folders
-internal/httpserve/handlers/fixtures/emails/*
+
+# emails tests 
+emails/*
 fixtures/emails/*
-pkg/httpsling/testdata/*
 
 # Packages
 *.7z

Taskfile.yaml

@@ -113,7 +113,7 @@ tasks:
     cmds:
       - task: compose:redis
       - task: compose:fga
-      - task: compose:kafka
+      - task: compose:riverboat
       - task: run-api
 
   run-api:

cmd/serve.go

@@ -6,8 +6,8 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/theopenlane/beacon/otelx"
+	"github.com/theopenlane/riverboat/pkg/riverqueue"
 
-	dbx "github.com/theopenlane/dbx/pkg/dbxclient"
 	"github.com/theopenlane/iam/fgax"
 
 	"github.com/theopenlane/utils/cache"
@@ -49,15 +49,12 @@ func serve(ctx context.Context) error {
 	serverOpts = append(serverOpts,
 		serveropts.WithConfigProvider(&config.ConfigProviderWithRefresh{}),
 		serveropts.WithHTTPS(),
-		serveropts.WithEmailManager(),
-		serveropts.WithTaskManager(),
+		serveropts.WithEmailConfig(),
 		serveropts.WithMiddleware(),
 		serveropts.WithRateLimiter(),
 		serveropts.WithSecureMW(),
 		serveropts.WithCacheHeaders(),
 		serveropts.WithCORS(),
-		serveropts.WithAnalytics(),
-		serveropts.WithEventPublisher(),
 	)
 
 	so := serveropts.NewServerOptions(serverOpts, k.String("config"))
@@ -97,13 +94,6 @@ func serve(ctx context.Context) error {
 		serveropts.WithSessionManager(redisClient),
 	)
 
-	// Setup DBx client
-	if so.Config.Settings.DBx.Enabled {
-		gc := so.Config.Settings.DBx.NewDefaultClient()
-
-		entOpts = append(entOpts, ent.DBx(gc.(*dbx.Client)))
-	}
-
 	// add otp manager, after redis is setup
 	so.AddServerOptions(
 		serveropts.WithOTP(),
@@ -113,32 +103,36 @@ func serve(ctx context.Context) error {
 	entOpts = append(
 		entOpts,
 		ent.Authz(*fgaClient),
-		ent.Emails(so.Config.Handler.EmailManager),
-		ent.Marionette(so.Config.Handler.TaskMan),
-		ent.Analytics(so.Config.Handler.AnalyticsClient),
 		ent.TOTP(so.Config.Handler.OTPManager),
 		ent.TokenManager(so.Config.Handler.TokenManager),
 		ent.SessionConfig(so.Config.Handler.SessionConfig),
 		ent.EntConfig(&so.Config.Settings.EntConfig),
+		ent.Emailer(&so.Config.Settings.Email),
 	)
 
 	// Setup DB connection
-	entdbClient, dbConfig, err := entdb.NewMultiDriverDBClient(ctx, so.Config.Settings.DB, entOpts)
+	log.Info().Interface("db", so.Config.Settings.DB).Msg("connecting to database")
+
+	jobOpts := []riverqueue.Option{
+		riverqueue.WithConnectionURI(so.Config.Settings.JobQueue.ConnectionURI),
+	}
+
+	dbClient, err := entdb.New(ctx, so.Config.Settings.DB, jobOpts, entOpts...)
 	if err != nil {
 		return err
 	}
 
-	defer entdbClient.Close()
+	defer dbClient.CloseAll() // nolint: errcheck
 
 	// Add Driver to the Handlers Config
-	so.Config.Handler.DBClient = entdbClient
+	so.Config.Handler.DBClient = dbClient
 
 	// Add redis client to Handlers Config
 	so.Config.Handler.RedisClient = redisClient
 
 	// add ready checks
 	so.AddServerOptions(
-		serveropts.WithReadyChecks(dbConfig, fgaClient, redisClient),
+		serveropts.WithReadyChecks(dbClient.Config, fgaClient, redisClient, dbClient.Job),
 	)
 
 	// add auth options
@@ -154,7 +148,7 @@ func serve(ctx context.Context) error {
 	srv := server.NewServer(so.Config)
 
 	// Setup Graph API Handlers
-	so.AddServerOptions(serveropts.WithGraphRoute(srv, entdbClient))
+	so.AddServerOptions(serveropts.WithGraphRoute(srv, dbClient))
 
 	if err := srv.StartEchoServer(ctx); err != nil {
 		log.Error().Err(err).Msg("failed to run server")

config/.env.example

@@ -85,18 +85,16 @@ CORE_AUTHZ_CREDENTIALS_ISSUER=""
 CORE_AUTHZ_CREDENTIALS_SCOPES=""
 CORE_DB_DEBUG="false"
 CORE_DB_DATABASENAME="openlane"
-CORE_DB_DRIVERNAME="libsql"
+CORE_DB_DRIVERNAME="pgx"
 CORE_DB_MULTIWRITE="false"
 CORE_DB_PRIMARYDBSOURCE="file:openlane.db"
 CORE_DB_SECONDARYDBSOURCE="file:backup.db"
 CORE_DB_CACHETTL="1s"
 CORE_DB_RUNMIGRATIONS="true"
 CORE_DB_MIGRATIONPROVIDER="atlas"
 CORE_DB_ENABLEHISTORY="false"
-CORE_DBX_ENABLED="true"
-CORE_DBX_BASEURL="http://localhost:1337"
-CORE_DBX_ENDPOINT="query"
-CORE_DBX_DEBUG="false"
+CORE_JOBQUEUE_CONNECTIONURI=""
+CORE_JOBQUEUE_RUNMIGRATIONS="false"
 CORE_REDIS_ENABLED="true"
 CORE_REDIS_ADDRESS="localhost:6379"
 CORE_REDIS_NAME="openlane"
@@ -121,24 +119,22 @@ CORE_TRACER_OTLP_CERTIFICATE=""
 CORE_TRACER_OTLP_HEADERS=""
 CORE_TRACER_OTLP_COMPRESSION=""
 CORE_TRACER_OTLP_TIMEOUT="10s"
-CORE_EMAIL_SENDGRIDAPIKEY=""
-CORE_EMAIL_FROMEMAIL="[email protected]"
-CORE_EMAIL_TESTING="true"
-CORE_EMAIL_ARCHIVE=""
-CORE_EMAIL_LISTID=""
-CORE_EMAIL_ADMINEMAIL="[email protected]"
-CORE_EMAIL_CONSOLEURL_CONSOLEBASE="https://console.theopenlane.io"
-CORE_EMAIL_CONSOLEURL_VERIFY="/verify"
-CORE_EMAIL_CONSOLEURL_INVITE="/invite"
-CORE_EMAIL_CONSOLEURL_RESET="/password-reset"
-CORE_EMAIL_MARKETINGURL_MARKETINGBASE="https://www.theopenlane.io"
-CORE_EMAIL_MARKETINGURL_SUBSCRIBERVERIFY="/verify"
+CORE_EMAIL_COMPANYNAME=""
+CORE_EMAIL_COMPANYADDRESS=""
+CORE_EMAIL_CORPORATION=""
+CORE_EMAIL_FROMEMAIL=""
+CORE_EMAIL_SUPPORTEMAIL=""
+CORE_EMAIL_LOGOURL=""
+CORE_EMAIL_URLS_ROOT=""
+CORE_EMAIL_URLS_PRODUCT=""
+CORE_EMAIL_URLS_DOCS=""
+CORE_EMAIL_URLS_VERIFY=""
+CORE_EMAIL_URLS_INVITE=""
+CORE_EMAIL_URLS_RESET=""
+CORE_EMAIL_URLS_VERIFYSUBSCRIBER=""
 CORE_SESSIONS_SIGNINGKEY="my-signing-secret"
 CORE_SESSIONS_ENCRYPTIONKEY="encryptionsecret"
 CORE_SESSIONS_DOMAIN=""
-CORE_POSTHOG_ENABLED="false"
-CORE_POSTHOG_APIKEY=""
-CORE_POSTHOG_HOST="https://app.posthog.com"
 CORE_TOTP_ENABLED="true"
 CORE_TOTP_CODELENGTH="6"
 CORE_TOTP_ISSUER=""
@@ -150,8 +146,3 @@ CORE_RATELIMIT_ENABLED="false"
 CORE_RATELIMIT_LIMIT="10"
 CORE_RATELIMIT_BURST="30"
 CORE_RATELIMIT_EXPIRES="10m"
-CORE_PUBLISHERCONFIG_ENABLED="false"
-CORE_PUBLISHERCONFIG_APPNAME="openlane"
-CORE_PUBLISHERCONFIG_ADDRESS="localhost:10000"
-CORE_PUBLISHERCONFIG_ADDRESSES="[localhost:10000]"
-CORE_PUBLISHERCONFIG_DEBUG="false"

config/config-dev.example.yaml

@@ -9,14 +9,24 @@ server:
   # tls settings
   tls:
     enabled: false
+
+# dbx settings
+dbx:
+  enabled: false
+
 # db settings
 db:
   debug: true
-  driverName: postgres
+  driverName: pgx
   primaryDbSource: "postgres://postgres:[email protected]:5432/core?sslmode=disable"
   multiWrite: false
   migrationProvider: "goose"
+  runMigrations: true
   enableHistory: true
+
+jobQueue:
+    connectionURI: "postgres://postgres:[email protected]:5432/jobs?sslmode=disable"
+
 # auth settings
 auth:
   enabled: true
@@ -48,6 +58,7 @@ auth:
       relyingPartyId: "localhost"
       requestOrigins:
         - "http://localhost:3001"
+
 # authz settings
 authz:
   enabled: true
@@ -61,22 +72,22 @@ authz:
 sessions:
   encryptionKey: encryptionsecret
   signingKey: my-signing-secret
+
 # email settings
 email:
-  testing: true
-  archive: "fixtures/email"
-  sendGridApiKey: "SG.FakeAPIKey"
-  consoleUrl:
-    consoleBase: "http://localhost:17608/"
-# analytics settings
-posthog:
-  enabled: false
-  apiKey: "phc_FakeKey"
-publisherConfig:
-    appName: openlane
-    addresses:
-        - localhost:10000
-    enabled: true
+    companyAddress: 5150 Broadway St · San Antonio, TX 78209
+    companyName: Openlane
+    corporation: theopenlane, Inc.
+    fromEmail: [email protected]
+    supportEmail: [email protected]
+    urls:
+      verifySubscriber: https://www.theopenlane.io/subscribe
+      verify: https://console.theopenlane.io/verify
+      reset: https://console.theopenlane.io/password-reset
+      invite: https://console.theopenlane.io/invite 
+      root: https://www.theopenlane.io
+      docs: https://docs.theopenlane.io
+      product: https://console.theopenlane.io
 
 entConfig:
   entityTypes:

config/config.example.yaml

@@ -55,49 +55,57 @@ db:
     cacheTTL: 1000000000
     databaseName: openlane
     debug: false
-    driverName: libsql
+    driverName: pgx
     enableHistory: false
     migrationProvider: atlas
     multiWrite: false
     primaryDbSource: file:openlane.db
     runMigrations: true
     secondaryDbSource: file:backup.db
-dbx:
-    baseUrl: http://localhost:1337
-    debug: false
-    enabled: true
-    endpoint: query
 email:
-    adminEmail: [email protected]
-    archive: ""
-    consoleUrl:
-        consoleBase: https://console.theopenlane.io
-        invite: /invite
-        reset: /password-reset
-        verify: /verify
-    fromEmail: [email protected]
-    listId: ""
-    marketingUrl:
-        marketingBase: https://www.theopenlane.io
-        subscriberVerify: /verify
-    sendGridApiKey: ""
-    testing: true
+    companyAddress: ""
+    companyName: ""
+    corporation: ""
+    fromEmail: ""
+    logoURL: ""
+    supportEmail: ""
+    urls:
+        docs: ""
+        invite: ""
+        product: ""
+        reset: ""
+        root: ""
+        verify: ""
+        verifySubscriber: ""
 entConfig:
     entityTypes: null
     flags:
         useListObjectServices: false
         useListUserService: true
-posthog:
-    apiKey: ""
-    enabled: false
-    host: https://app.posthog.com
-publisherConfig:
-    address: localhost:10000
-    addresses:
-        - localhost:10000
-    appName: openlane
-    debug: false
-    enabled: false
+jobQueue:
+    connectionURI: ""
+    riverConf:
+        AdvisoryLockPrefix: 0
+        CancelledJobRetentionPeriod: 0
+        CompletedJobRetentionPeriod: 0
+        DiscardedJobRetentionPeriod: 0
+        ErrorHandler: null
+        FetchCooldown: 0
+        FetchPollInterval: 0
+        ID: ""
+        JobCleanerTimeout: 0
+        JobTimeout: 0
+        Logger: null
+        MaxAttempts: 0
+        PeriodicJobs: null
+        PollOnly: false
+        Queues: null
+        ReindexerSchedule: null
+        RescueStuckJobsAfter: 0
+        RetryPolicy: null
+        TestOnly: false
+        Workers: null
+    runMigrations: false
 ratelimit:
     burst: 30
     enabled: false

config/config.go

@@ -11,12 +11,12 @@ import (
 	"github.com/knadh/koanf/v2"
 	"github.com/mcuadros/go-defaults"
 	"github.com/theopenlane/beacon/otelx"
-	dbx "github.com/theopenlane/dbx/pkg/dbxclient"
+	"github.com/theopenlane/emailtemplates"
 	"github.com/theopenlane/entx"
 	"github.com/theopenlane/iam/fgax"
+	"github.com/theopenlane/riverboat/pkg/riverqueue"
 
 	"github.com/theopenlane/iam/totp"
-	"github.com/theopenlane/utils/emails"
 
 	"github.com/theopenlane/iam/sessions"
 	"github.com/theopenlane/iam/tokens"
@@ -25,8 +25,6 @@ import (
 
 	"github.com/theopenlane/core/internal/ent/entconfig"
 	"github.com/theopenlane/core/internal/httpserve/handlers"
-	"github.com/theopenlane/core/pkg/analytics/posthog"
-	"github.com/theopenlane/core/pkg/events/kafka/kafkaconfig"
 	"github.com/theopenlane/core/pkg/middleware/cachecontrol"
 	"github.com/theopenlane/core/pkg/middleware/cors"
 	"github.com/theopenlane/core/pkg/middleware/mime"
@@ -53,24 +51,20 @@ type Config struct {
 	Authz fgax.Config `json:"authz" koanf:"authz"`
 	// DB contains the database configuration for the ent client
 	DB entx.Config `json:"db" koanf:"db"`
-	// DBx contains the dbx client configuration
-	DBx dbx.Config `json:"dbx" koanf:"dbx"`
+	// JobQueue contains the configuration for the job queue (river) client
+	JobQueue riverqueue.Config `json:"jobQueue" koanf:"jobQueue"`
 	// Redis contains the redis configuration for the key-value store
 	Redis cache.Config `json:"redis" koanf:"redis"`
 	// Tracer contains the tracing config for opentelemetry
 	Tracer otelx.Config `json:"tracer" koanf:"tracer"`
 	// Email contains email sending configuration for the server
-	Email emails.Config `json:"email" koanf:"email"`
+	Email emailtemplates.Config `json:"email" koanf:"email"`
 	// Sessions config for user sessions and cookies
 	Sessions sessions.Config `json:"sessions" koanf:"sessions"`
-	// PostHog contains the configuration for the PostHog analytics
-	PostHog posthog.Config `json:"posthog" koanf:"posthog"`
 	// TOTP contains the configuration for the TOTP provider
 	TOTP totp.Config `json:"totp" koanf:"totp"`
 	// Ratelimit contains the configuration for the rate limiter
 	Ratelimit ratelimit.Config `json:"ratelimit" koanf:"ratelimit"`
-	// EventPublisher contains the configuration for the event publisher
-	Events kafkaconfig.Config `json:"publisherConfig" koanf:"publisherConfig"`
 }
 
 // Server settings for the echo server