feat(vacuum): add vacuum management for cleaning expired packages

go.mod

@@ -48,6 +48,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
+	github.com/dustin/go-humanize v1.0.1
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/gdamore/tcell/v2 v2.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
@@ -82,6 +83,7 @@ require (
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.etcd.io/bbolt v1.3.11
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/sync v0.9.0 // indirect

go.sum

@@ -56,6 +56,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
 github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
@@ -255,6 +257,8 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
+go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

pkg/cli/runner.go

@@ -20,6 +20,7 @@ import (
 	"github.com/aquaproj/aqua/v2/pkg/cli/update"
 	"github.com/aquaproj/aqua/v2/pkg/cli/updateaqua"
 	"github.com/aquaproj/aqua/v2/pkg/cli/util"
+	"github.com/aquaproj/aqua/v2/pkg/cli/vacuum"
 	"github.com/aquaproj/aqua/v2/pkg/cli/version"
 	"github.com/aquaproj/aqua/v2/pkg/cli/which"
 	"github.com/urfave/cli/v2"
@@ -105,6 +106,7 @@ func Run(ctx context.Context, param *util.Param, args ...string) error { //nolin
 			upc.New,
 			remove.New,
 			update.New,
+			vacuum.New,
 		),
 	}
 

pkg/cli/util/util.go

@@ -33,15 +33,24 @@ type LDFlags struct {
 	Date    string
 }
 
-func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, ldFlags *LDFlags) error { //nolint:funlen,cyclop
+func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, ldFlags *LDFlags) error {
 	wd, err := os.Getwd()
 	if err != nil {
 		return fmt.Errorf("get the current directory: %w", err)
 	}
 	param.Args = c.Args().Slice()
-	if logLevel := c.String("log-level"); logLevel != "" {
-		param.LogLevel = logLevel
+	setBasicParams(c, logE, commandName, param, wd, ldFlags)
+	setLogParams(c, param, logE)
+	if err := setEnvParams(param); err != nil {
+		return fmt.Errorf("error during setting params from Env vars: %w", err)
+	}
+	if err := setChecksumParams(param); err != nil {
+		return fmt.Errorf("error during setting params from checksum params: %w", err)
 	}
+	return nil
+}
+
+func setBasicParams(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, wd string, ldFlags *LDFlags) {
 	param.ConfigFilePath = c.String("config")
 	param.Dest = c.String("o")
 	param.OutTestData = c.String("out-testdata")
@@ -66,14 +75,11 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con
 	if cmd := c.String("cmd"); cmd != "" {
 		param.Commands = strings.Split(cmd, ",")
 	}
-	param.LogColor = os.Getenv("AQUA_LOG_COLOR")
 	param.AQUAVersion = ldFlags.Version
 	param.AquaCommitHash = ldFlags.Commit
 	param.RootDir = config.GetRootDir(osenv.New())
 	homeDir, _ := os.UserHomeDir()
 	param.HomeDir = homeDir
-	log.SetLevel(param.LogLevel, logE)
-	log.SetColor(param.LogColor, logE)
 	param.MaxParallelism = config.GetMaxParallelism(os.Getenv("AQUA_MAX_PARALLELISM"), logE)
 	param.GlobalConfigFilePaths = finder.ParseGlobalConfigFilePaths(wd, os.Getenv("AQUA_GLOBAL_CONFIG"))
 	param.Deep = c.Bool("deep")
@@ -84,7 +90,18 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con
 	param.ProgressBar = os.Getenv("AQUA_PROGRESS_BAR") == "true"
 	param.Tags = parseTags(strings.Split(c.String("tags"), ","))
 	param.ExcludedTags = parseTags(strings.Split(c.String("exclude-tags"), ","))
+}
 
+func setLogParams(c *cli.Context, param *config.Param, logE *logrus.Entry) {
+	if logLevel := c.String("log-level"); logLevel != "" {
+		param.LogLevel = logLevel
+	}
+	param.LogColor = os.Getenv("AQUA_LOG_COLOR")
+	log.SetLevel(param.LogLevel, logE)
+	log.SetColor(param.LogColor, logE)
+}
+
+func setEnvParams(param *config.Param) error { //nolint:cyclop
 	if a := os.Getenv("AQUA_DISABLE_LAZY_INSTALL"); a != "" {
 		disableLazyInstall, err := strconv.ParseBool(a)
 		if err != nil {
@@ -108,6 +125,17 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con
 			}
 		}
 	}
+	if a := os.Getenv("AQUA_VACUUM_DAYS"); a != "" {
+		vacuumDays, err := strconv.Atoi(a)
+		if err != nil || vacuumDays <= 0 {
+			return fmt.Errorf("parse the environment variable AQUA_VACUUM_DAYS as a positive integer: %w", err)
+		}
+		param.VacuumDays = &vacuumDays
+	}
+	return nil
+}
+
+func setChecksumParams(param *config.Param) error {
 	if a := os.Getenv("AQUA_CHECKSUM"); a != "" {
 		chksm, err := strconv.ParseBool(a)
 		if err != nil {

pkg/cli/vacuum/command.go

@@ -0,0 +1,116 @@
+package vacuum
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/aquaproj/aqua/v2/pkg/cli/profile"
+	"github.com/aquaproj/aqua/v2/pkg/cli/util"
+	"github.com/aquaproj/aqua/v2/pkg/config"
+	"github.com/aquaproj/aqua/v2/pkg/controller"
+	"github.com/urfave/cli/v2"
+)
+
+const description = `Perform vacuuming tasks.
+If no argument is provided, the vacuum will clean expired packages.
+
+	# Execute vacuum cleaning
+	$ aqua vacuum
+
+This command has an alias "v".
+
+	$ aqua v
+
+Enable vacuuming by setting the AQUA_VACUUM_DAYS environment variable to a value greater than 0.
+This command removes versions of packages that have not been used for the specified number of days.
+
+You can list all packages managed by the vacuum system or only expired packages.
+
+	# List all packages managed by the vacuum system
+	$ aqua vacuum --list
+	$ aqua vacuum -l
+
+	# List only expired packages
+	$ aqua vacuum --expired
+	$ aqua vacuum -e
+
+`
+
+type command struct {
+	r *util.Param
+}
+
+func New(r *util.Param) *cli.Command {
+	i := &command{
+		r: r,
+	}
+	return &cli.Command{
+		Name:        "vacuum",
+		Usage:       "Operate vacuuming tasks (If AQUA_VACUUM_DAYS is set)",
+		Aliases:     []string{"v"},
+		Description: description,
+		Action:      i.action,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:     "list",
+				Usage:    "List all packages managed by vacuum system",
+				Category: "list",
+			},
+			&cli.BoolFlag{
+				Name:     "expired",
+				Usage:    "List only expired packages",
+				Category: "list",
+			},
+		},
+	}
+}
+
+// Define the vacuum modes for the CLI
+const (
+	ListPackages          string = "list-packages"
+	ListExpiredPackages   string = "list-expired-packages"
+	VacuumExpiredPackages string = "vacuum-expired-packages"
+)
+
+func (i *command) action(c *cli.Context) error {
+	profiler, err := profile.Start(c)
+	if err != nil {
+		return fmt.Errorf("start CPU Profile or tracing: %w", err)
+	}
+	defer profiler.Stop()
+
+	mode := parseVacuumMode(c)
+
+	param := &config.Param{}
+	if err := util.SetParam(c, i.r.LogE, "vacuum", param, i.r.LDFlags); err != nil {
+		return fmt.Errorf("parse the command line arguments: %w", err)
+	}
+
+	if param.VacuumDays == nil {
+		return errors.New("vacuum is not enabled, please set the AQUA_VACUUM_DAYS environment variable")
+	}
+
+	ctrl := controller.InitializeVacuumCommandController(c.Context, param, http.DefaultClient, i.r.Runtime)
+
+	vacuumMode, err := ctrl.GetVacuumModeCLI(mode)
+	if err != nil {
+		return fmt.Errorf("get vacuum mode: %w", err)
+	}
+
+	if err := ctrl.Vacuum(c.Context, i.r.LogE, vacuumMode, nil); err != nil {
+		return fmt.Errorf("vacuum: %w", err)
+	}
+	return nil
+}
+
+func parseVacuumMode(c *cli.Context) string {
+	mode := VacuumExpiredPackages
+	if c.Bool("list") {
+		mode = ListPackages
+	}
+	if c.Bool("expired") {
+		mode = ListExpiredPackages
+	}
+	return mode
+}

pkg/config/package.go

@@ -294,6 +294,7 @@ type Param struct {
 	Installed                         bool
 	PolicyConfigFilePaths             []string
 	Commands                          []string
+	VacuumDays                        *int // When defined, vacuuming is enabled
 }
 
 func appendExt(s, format string) string {

pkg/controller/exec/controller.go

@@ -7,6 +7,7 @@ import (
 	"runtime"
 
 	"github.com/aquaproj/aqua/v2/pkg/config"
+	"github.com/aquaproj/aqua/v2/pkg/controller/vacuum"
 	"github.com/aquaproj/aqua/v2/pkg/controller/which"
 	"github.com/aquaproj/aqua/v2/pkg/installpackage"
 	"github.com/aquaproj/aqua/v2/pkg/osexec"
@@ -26,13 +27,14 @@ type Controller struct {
 	fs               afero.Fs
 	policyReader     PolicyReader
 	enabledXSysExec  bool
+	vacuumCtrl       *vacuum.Controller
 }
 
 type Installer interface {
 	InstallPackage(ctx context.Context, logE *logrus.Entry, param *installpackage.ParamInstallPackage) error
 }
 
-func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader) *Controller {
+func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader, vacuumCtrl *vacuum.Controller) *Controller {
 	return &Controller{
 		stdin:            os.Stdin,
 		stdout:           os.Stdout,
@@ -43,6 +45,7 @@ func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, o
 		enabledXSysExec:  getEnabledXSysExec(osEnv, runtime.GOOS),
 		fs:               fs,
 		policyReader:     policyReader,
+		vacuumCtrl:       vacuumCtrl,
 	}
 }
 

pkg/controller/exec/exec.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/aquaproj/aqua/v2/pkg/checksum"
 	"github.com/aquaproj/aqua/v2/pkg/config"
+	"github.com/aquaproj/aqua/v2/pkg/controller/vacuum"
 	"github.com/aquaproj/aqua/v2/pkg/controller/which"
 	"github.com/aquaproj/aqua/v2/pkg/installpackage"
 	"github.com/aquaproj/aqua/v2/pkg/osexec"
@@ -62,9 +63,22 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config
 	if err := c.install(ctx, logE, findResult, policyCfgs, param); err != nil {
 		return err
 	}
+	c.vacuumClose(ctx, logE) // Ensure that the vacuum process and db are closed
 	return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...)
 }
 
+func (c *Controller) vacuumClose(ctx context.Context, logE *logrus.Entry) {
+	if c.vacuumCtrl == nil {
+		return
+	}
+	if err := c.vacuumCtrl.Vacuum(ctx, logE, vacuum.Close, nil); err != nil {
+		// If the closing vacuum db failed, we should not stop the process
+		// so we log the error and continue the process.
+		// Updating vacuum db will be retried next time.
+		logE.WithError(err).Error("close the vacuum db failed")
+	}
+}
+
 func (c *Controller) install(ctx context.Context, logE *logrus.Entry, findResult *which.FindResult, policies []*policy.Config, param *config.Param) error {
 	var checksums *checksum.Checksums
 	if findResult.Config.ChecksumEnabled(param.EnforceChecksum, param.Checksum) {