Add purge support (#4)

metal-stack · Feb 21, 2024 · ccfa00d · ccfa00d
1 parent 5412b22
commit ccfa00d
Show file tree

Hide file tree

Showing 17 changed files with 962 additions and 267 deletions.
diff --git a/README.md b/README.md
@@ -28,6 +28,7 @@ kubectl apply -f deploy
 
 ## TODO
 
+- [x] support purging
 - [ ] eventually support http(s) artifacts to be stored as OCIs
-- [ ] support Regex Match for image tags
-- [ ] store a OCI artifact which reflects all stored images
+- [ ] ~~~support Regex Match for image tags~~~
+- [ ] store a OCI artifact which reflects all stored images ?
diff --git a/api/v1/types.go b/api/v1/types.go
@@ -37,6 +37,8 @@ type ImageMirror struct {
 	Destination string `json:"destination,omitempty"`
 	// Match defines which images to mirror
 	Match Match `json:"match,omitempty"`
+	// Purge defines which images should be purged
+	Purge *Purge `json:"purge,omitempty"`
 }
 
 type Match struct {
@@ -50,6 +52,16 @@ type Match struct {
 	Last *int64 `json:"last,omitempty"`
 }
 
+type Purge struct {
+	// Tags is a exact list of tags to purge
+	Tags []string `json:"tags,omitempty"`
+	// Semver defines a semantic version of tags to purge
+	Semver *string `json:"semver,omitempty"`
+	// NoMatch if set to true, all images which are not matched by the Match specification will be purged.
+	// latest will never be purged
+	NoMatch bool `json:"no_match,omitempty"`
+}
+
 func (c Config) Validate() error {
 	var errs []error
 	sources := make(map[string]bool)
@@ -92,12 +104,22 @@ func (c Config) Validate() error {
 		}
 
 		if image.Match.Semver != nil {
-			_, err := semver.NewConstraint(*image.Match.Semver)
-			if err != nil {
+			if _, err := semver.NewConstraint(*image.Match.Semver); err != nil {
 				errs = append(errs, fmt.Errorf("image.match.semver is invalid, image source:%q, semver:%q %w", image.Source, *image.Match.Semver, err))
 			}
 		}
 
+		if image.Purge != nil {
+			if image.Purge.Semver != nil {
+				if _, err := semver.NewConstraint(*image.Purge.Semver); err != nil {
+					errs = append(errs, fmt.Errorf("image.purge.semver is invalid, image source:%q, semver:%q %w", image.Source, *image.Purge.Semver, err))
+				}
+			}
+			if image.Purge.NoMatch && image.Match.AllTags {
+				errs = append(errs, fmt.Errorf("image.purge.nomatch and image.match.alltags cannot be set both image source:%q", image.Source))
+			}
+		}
+
 		srcRef, err := name.ParseReference(image.Source)
 		if err != nil {
 			errs = append(errs, err)

diff --git a/api/v1/types_test.go b/api/v1/types_test.go
@@ -52,7 +52,7 @@ func TestConfig_Validate(t *testing.T) {
 			wantErr: true,
 		},
 		{
-			name: "invalid semver",
+			name: "invalid match semver",
 			Images: []ImageMirror{
 				{
 					Source:      "abc",
@@ -64,6 +64,19 @@ func TestConfig_Validate(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "invalid purge semver",
+			Images: []ImageMirror{
+				{
+					Source:      "abc",
+					Destination: "abc",
+					Purge: &Purge{
+						Semver: pointer.Pointer("abc"),
+					},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "image cde is used in two images",
 			Images: []ImageMirror{
@@ -108,6 +121,22 @@ func TestConfig_Validate(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "invalid purge and alltags set",
+			Images: []ImageMirror{
+				{
+					Source:      "abc",
+					Destination: "abc",
+					Match: Match{
+						AllTags: true,
+					},
+					Purge: &Purge{
+						NoMatch: true,
+					},
+				},
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/cmd/main.go b/cmd/main.go
@@ -57,13 +57,91 @@ var (
 			}
 
 			s := newServer(log, config)
-			if err := s.run(); err != nil {
+			if err := s.mirror(); err != nil {
 				log.Error("error during mirror", "error", err)
 				os.Exit(1)
 			}
 			return nil
 		},
 	}
+	purgeCmd = &cli.Command{
+		Name:  "purge",
+		Usage: "purge images as specified in configuration",
+		Flags: []cli.Flag{
+			debugFlag,
+			configMapFlag,
+		},
+		Action: func(ctx *cli.Context) error {
+			level := slog.LevelInfo
+			if ctx.Bool(debugFlag.Name) {
+				level = slog.LevelDebug
+			}
+			jsonHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})
+			log := slog.New(jsonHandler)
+
+			log.Info("start purge", "version", v.V.String())
+			raw, err := os.ReadFile(ctx.String(configMapFlag.Name))
+			if err != nil {
+				return fmt.Errorf("unable to read config file:%w", err)
+			}
+			var config apiv1.Config
+			err = yaml.Unmarshal(raw, &config)
+			if err != nil {
+				return fmt.Errorf("unable to parse config file:%w", err)
+			}
+
+			err = config.Validate()
+			if err != nil {
+				return fmt.Errorf("config invalid:%w", err)
+			}
+
+			s := newServer(log, config)
+			if err := s.purge(); err != nil {
+				log.Error("error during purge", "error", err)
+				os.Exit(1)
+			}
+			return nil
+		},
+	}
+	purgeUnknwonCmd = &cli.Command{
+		Name:  "purge-unknown",
+		Usage: "purge unknown images according to the configuration",
+		Flags: []cli.Flag{
+			debugFlag,
+			configMapFlag,
+		},
+		Action: func(ctx *cli.Context) error {
+			level := slog.LevelInfo
+			if ctx.Bool(debugFlag.Name) {
+				level = slog.LevelDebug
+			}
+			jsonHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})
+			log := slog.New(jsonHandler)
+
+			log.Info("start purge unknown", "version", v.V.String())
+			raw, err := os.ReadFile(ctx.String(configMapFlag.Name))
+			if err != nil {
+				return fmt.Errorf("unable to read config file:%w", err)
+			}
+			var config apiv1.Config
+			err = yaml.Unmarshal(raw, &config)
+			if err != nil {
+				return fmt.Errorf("unable to parse config file:%w", err)
+			}
+
+			err = config.Validate()
+			if err != nil {
+				return fmt.Errorf("config invalid:%w", err)
+			}
+
+			s := newServer(log, config)
+			if err := s.purgeUnknown(); err != nil {
+				log.Error("error during purge", "error", err)
+				os.Exit(1)
+			}
+			return nil
+		},
+	}
 )
 
 func main() {
@@ -72,6 +150,8 @@ func main() {
 		Usage: "oci mirror server",
 		Commands: []*cli.Command{
 			mirrorCmd,
+			purgeCmd,
+			purgeUnknwonCmd,
 		},
 	}
 

diff --git a/cmd/server.go b/cmd/server.go
@@ -7,7 +7,7 @@ import (
 	"time"
 
 	apiv1 "github.com/metal-stack/oci-mirror/api/v1"
-	"github.com/metal-stack/oci-mirror/pkg/mirror"
+	"github.com/metal-stack/oci-mirror/pkg/container"
 )
 
 type server struct {
@@ -22,9 +22,9 @@ func newServer(log *slog.Logger, config apiv1.Config) *server {
 	}
 }
 
-func (s *server) run() error {
+func (s *server) mirror() error {
 	start := time.Now()
-	m := mirror.New(s.log, s.config)
+	m := container.New(s.log.WithGroup("mirror"), s.config)
 	err := m.Mirror(context.Background())
 	if err != nil {
 		s.log.Error(fmt.Sprintf("error mirroring images, duration %s", time.Since(start)), "error", err)
@@ -33,3 +33,27 @@ func (s *server) run() error {
 	s.log.Info(fmt.Sprintf("finished mirroring after %s", time.Since(start)))
 	return nil
 }
+
+func (s *server) purge() error {
+	start := time.Now()
+	m := container.New(s.log.WithGroup("purge"), s.config)
+	err := m.Purge(context.Background())
+	if err != nil {
+		s.log.Error(fmt.Sprintf("error purging images, duration %s", time.Since(start)), "error", err)
+		return err
+	}
+	s.log.Info(fmt.Sprintf("finished purging after %s", time.Since(start)))
+	return nil
+}
+
+func (s *server) purgeUnknown() error {
+	start := time.Now()
+	m := container.New(s.log.WithGroup("purgeunknown"), s.config)
+	err := m.PurgeUnknown(context.Background())
+	if err != nil {
+		s.log.Error(fmt.Sprintf("error purging unknown images, duration %s", time.Since(start)), "error", err)
+		return err
+	}
+	s.log.Info(fmt.Sprintf("finished purging unknown after %s", time.Since(start)))
+	return nil
+}
diff --git a/deploy/oci-mirror.yaml b/deploy/oci-mirror.yaml
@@ -34,6 +34,68 @@ spec:
                     path: oci-mirror.yaml
           restartPolicy: OnFailure
 ---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: oci-mirror-purge
+  namespace: mirror
+spec:
+  schedule: "*/40 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: oci-mirror
+            image: ghcr.io/metal-stack/oci-mirror
+            imagePullPolicy: IfNotPresent
+            args:
+            - purge
+            - --mirror-config=/config/oci-mirror.yaml
+            volumeMounts:
+              - name: mirror-config
+                mountPath: /config
+          volumes:
+            - name: mirror-config
+              secret:
+                secretName: mirror-config
+                items:
+                  - key: oci-mirror.yaml
+                    path: oci-mirror.yaml
+          restartPolicy: OnFailure
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: oci-mirror-purge-unknown
+  namespace: mirror
+spec:
+  # once a week on every monday at 2:00 o'clock
+  schedule: "0 2 * * 1"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: oci-mirror
+            image: ghcr.io/metal-stack/oci-mirror
+            imagePullPolicy: IfNotPresent
+            args:
+            - purge-unknown
+            - --mirror-config=/config/oci-mirror.yaml
+            volumeMounts:
+              - name: mirror-config
+                mountPath: /config
+          volumes:
+            - name: mirror-config
+              secret:
+                secretName: mirror-config
+                items:
+                  - key: oci-mirror.yaml
+                    path: oci-mirror.yaml
+          restartPolicy: OnFailure
+
+---
 apiVersion: v1
 kind: Secret
 metadata: