Add batch creation logic for the reminder service

Signed-off-by: Vyom-Yadav <[email protected]>

cmd/reminder/app/start.go

@@ -49,11 +49,9 @@ func start(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("unable to read config: %w", err)
 	}
 
-	// Configuration is normalized to ensure that all values are valid
-	// and if they can't be fixed, an error is returned
-	_, err = cfg.Normalize(cmd)
+	err = cfg.Validate()
 	if err != nil {
-		return fmt.Errorf("unable to normalize config: %w", err)
+		return fmt.Errorf("error validating config: %w", err)
 	}
 
 	ctx = logger.FromFlags(cfg.LoggingConfig).WithContext(ctx)

config/reminder-config.yaml.example

@@ -16,8 +16,6 @@
 recurrence:
   interval: "1h"
   batch_size: 100
-  max_per_project: 10
-  min_project_fetch_limit: 10
   min_elapsed: "1h"
 
 database:
@@ -28,8 +26,6 @@ database:
   dbname: minder
   sslmode: disable
 
-cursor_file: "/tmp/reminder-cursor"
-
 logging
   level: "debug"
 

database/migrations/000060_repo_reminder.down.sql

@@ -0,0 +1,17 @@
+-- Copyright 2024 Stacklok, Inc
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+ALTER TABLE repositories DROP COLUMN reminder_last_sent;
+
+DROP EXTENSION IF EXISTS tsm_system_rows;

database/migrations/000060_repo_reminder.up.sql

@@ -0,0 +1,17 @@
+-- Copyright 2024 Stacklok, Inc
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+ALTER TABLE repositories ADD COLUMN reminder_last_sent TIMESTAMP;
+
+CREATE EXTENSION IF NOT EXISTS tsm_system_rows;

database/query/repositories.sql

@@ -16,6 +16,10 @@ INSERT INTO repositories (
     provider_id
     ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, sqlc.arg(default_branch), sqlc.arg(license), sqlc.arg(provider_id)) RETURNING *;
 
+-- name: GetRandomRepository :one
+SELECT * FROM repositories
+TABLESAMPLE SYSTEM_ROWS(1);
+
 -- name: GetRepositoryByRepoID :one
 SELECT * FROM repositories WHERE repo_id = $1;
 
@@ -45,6 +49,28 @@ WHERE project_id = $1 AND webhook_id IS NOT NULL
     AND (lower(provider) = lower(sqlc.narg('provider')::text) OR sqlc.narg('provider')::text IS NULL)
 ORDER BY repo_name;
 
+-- name: ListEligibleRepositoriesAfterID :many
+SELECT r.* FROM repositories r
+  INNER JOIN rule_evaluations re ON re.repository_id = r.id
+  INNER JOIN rule_details_eval rde ON rde.rule_eval_id = re.id
+WHERE r.id > $1
+GROUP BY r.id
+HAVING MIN(rde.last_updated) + sqlc.arg('min_elapsed')::interval < NOW()
+ORDER BY r.id
+LIMIT sqlc.narg('limit')::bigint;
+
+-- name: UpdateReminderLastSentById :exec
+UPDATE repositories
+SET reminder_last_sent = NOW()
+WHERE id = $1;
+
+-- name: RepositoryExistsAfterID :one
+SELECT EXISTS (
+  SELECT 1
+  FROM repositories
+  WHERE id > $1)
+AS exists;
+
 -- name: DeleteRepository :exec
 DELETE FROM repositories
 WHERE id = $1;

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/itchyny/gojq v0.12.15
+	github.com/jackc/pgtype v1.14.0
 	github.com/lib/pq v1.10.9
 	github.com/motemen/go-loghttp v0.0.0-20231107055348-29ae44b293f4
 	github.com/oapi-codegen/runtime v1.1.1
@@ -114,6 +115,7 @@ require (
 	github.com/gorilla/schema v1.2.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.5 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
 	github.com/jackc/pgx/v5 v5.5.5 // indirect