feat: use key-value store for analysis data

Makefile

@@ -98,6 +98,9 @@ coverage-html: coverage
 gobench:
 	go test -bench=. $(PACKAGE)/pkg/analyze
 
+heap-profile:
+	go tool pprof -web http://localhost:6060/debug/pprof/heap
+
 benchmark:
 	sudo cpupower frequency-set -g performance
 	hyperfine --export-markdown=bench-cold.md \
@@ -133,5 +136,6 @@ release:
 
 install-dev-dependencies:
 	go install gotest.tools/gotestsum@latest
+	go install github.com/mitchellh/gox@latest
 
 .PHONY: run build build-static build-all test gobench benchmark coverage coverage-html clean clean-uncompressed-dist man show-man release

README.md

@@ -69,7 +69,7 @@ Using curl:
 
     winget install gdu
 
-You can either run it as `gdu_windows_amd64.exe` or 
+You can either run it as `gdu_windows_amd64.exe` or
 * add an alias with `Doskey`.
 * add `alias gdu="gdu_windows_amd64.exe"` to your `~/.bashrc` file if using Git Bash to run it as `gdu`.
 
@@ -181,11 +181,14 @@ Flags:
   -p, --no-progress                   Do not show progress in non-interactive mode
   -n, --non-interactive               Do not run in interactive mode
   -o, --output-file string            Export all info into file as JSON
+  -r, --read-from-storage             Read analysis data from persistent key-value storage
   -a, --show-apparent-size            Show apparent size
   -d, --show-disks                    Show all mounted disks
   -B, --show-relative-size            Show relative size
       --si                            Show sizes with decimal SI prefixes (kB, MB, GB) instead of binary prefixes (KiB, MiB, GiB)
+      --storage-path string           Path to persistent key-value storage directory (default is /tmp/badger) (default "/tmp/badger")
   -s, --summarize                     Show only a total in non-interactive mode
+      --use-storage                   Use persistent key-value storage for analysis data (experimental)
   -v, --version                       Print version
       --write-config                  Write current configuration to file (default is $HOME/.gdu.yaml)
 
@@ -221,6 +224,9 @@ In interactive mode:
     gdu -o- / | gzip -c >report.json.gz   # write all info to JSON file for later analysis
     zcat report.json.gz | gdu -f-         # read analysis from file
 
+    GOGC=10 gdu -g --use-storage /        # use persistent key-value storage for saving analysis data
+    gdu -r /                              # read saved analysis data from persistent key-value storage
+
 ## Modes
 
 Gdu has three modes: interactive (default), non-interactive and export.
@@ -320,6 +326,18 @@ Example running gdu with constant GC, but not so aggressive as default:
 GOGC=200 gdu -g /
 ```
 
+## Saving analysis data to persistent key-value storage (experimental)
+
+Gdu can store the analysis data to persistent key-value storage instead of just memory.
+Gdu will run much slower (approx 10x) but it should use much less memory (when using small GOGC as well).
+Gdu can also reopen with the saved data.
+Currently only BadgerDB is supported as the key-value storage (embedded).
+
+```
+GOGC=10 gdu -g --use-storage /    # saves analysis data to key-value storage
+gdu -r /                          # reads just saved data, does not run analysis again
+```
+
 ## Running tests
 
     make install-dev-dependencies

cmd/gdu/app/app.go

@@ -16,6 +16,7 @@
 
 	"github.com/dundee/gdu/v5/build"
 	"github.com/dundee/gdu/v5/internal/common"
+	"github.com/dundee/gdu/v5/pkg/analyze"
 	"github.com/dundee/gdu/v5/pkg/device"
 	gfs "github.com/dundee/gdu/v5/pkg/fs"
 	"github.com/dundee/gdu/v5/report"
@@ -30,11 +31,13 @@
 	ListDevices(getter device.DevicesInfoGetter) error
 	AnalyzePath(path string, parentDir gfs.Item) error
 	ReadAnalysis(input io.Reader) error
+	ReadFromStorage(storagePath, path string) error
 	SetIgnoreDirPaths(paths []string)
 	SetIgnoreDirPatterns(paths []string) error
 	SetIgnoreFromFile(ignoreFile string) error
 	SetIgnoreHidden(value bool)
 	SetFollowSymlinks(value bool)
+	SetAnalyzer(analyzer common.Analyzer)
 	StartUILoop() error
 }
 
@@ -61,6 +64,9 @@
 	FollowSymlinks    bool     `yaml:"follow-symlinks"`
 	Profiling         bool     `yaml:"profiling"`
 	ConstGC           bool     `yaml:"const-gc"`
+	UseStorage        bool     `yaml:"use-storage"`
+	StoragePath       string   `yaml:"storage-path"`
+	ReadFromStorage   bool     `yaml:"read-from-storage"`
 	Summarize         bool     `yaml:"summarize"`
 	UseSIPrefix       bool     `yaml:"use-si-prefix"`
 	NoPrefix          bool     `yaml:"no-prefix"`
@@ -135,6 +141,12 @@
 		return
 	}
 
+	if a.Flags.UseStorage {
+		ui.SetAnalyzer(analyze.CreateStoredAnalyzer(a.Flags.StoragePath))
+	}
+	if a.Flags.FollowSymlinks {
+		ui.SetFollowSymlinks(true)
+	}
 	if err = a.setNoCross(path); err != nil {
 		return
 	}
@@ -273,10 +285,6 @@
 		tview.Styles.BorderColor = tcell.ColorDefault
 	}
 
-	if a.Flags.FollowSymlinks {
-		ui.SetFollowSymlinks(true)
-	}
-
 	return ui, nil
 }
 
@@ -324,6 +332,11 @@
 		if err := ui.ReadAnalysis(input); err != nil {
 			return fmt.Errorf("reading analysis: %w", err)
 		}
+	} else if a.Flags.ReadFromStorage {
+		ui.SetAnalyzer(analyze.CreateStoredAnalyzer(a.Flags.StoragePath))
+		if err := ui.ReadFromStorage(a.Flags.StoragePath, path); err != nil {
+			return fmt.Errorf("reading from storage (%s): %w", a.Flags.StoragePath, err)
+		}
 	} else {
 		if build.RootPathPrefix != "" {
 			path = build.RootPathPrefix + path

cmd/gdu/main.go

@@ -57,6 +57,10 @@ func init() {
 	flags.BoolVarP(&af.ConstGC, "const-gc", "g", false, "Enable memory garbage collection during analysis with constant level set by GOGC")
 	flags.BoolVar(&af.Profiling, "enable-profiling", false, "Enable collection of profiling data and provide it on http://localhost:6060/debug/pprof/")
 
+	flags.BoolVar(&af.UseStorage, "use-storage", false, "Use persistent key-value storage for analysis data (experimental)")
+	flags.StringVar(&af.StoragePath, "storage-path", "/tmp/badger", "Path to persistent key-value storage directory (default is /tmp/badger)")
+	flags.BoolVarP(&af.ReadFromStorage, "read-from-storage", "r", false, "Read analysis data from persistent key-value storage")
+
 	flags.BoolVarP(&af.ShowDisks, "show-disks", "d", false, "Show all mounted disks")
 	flags.BoolVarP(&af.ShowApparentSize, "show-apparent-size", "a", false, "Show apparent size")
 	flags.BoolVarP(&af.ShowRelativeSize, "show-relative-size", "B", false, "Show relative size")

go.mod

@@ -1,33 +1,48 @@
 module github.com/dundee/gdu/v5
 
-go 1.18
+go 1.20
 
 require (
+	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/fatih/color v1.15.0
 	github.com/gdamore/tcell/v2 v2.6.0
 	github.com/maruel/natural v1.1.0
 	github.com/mattn/go-isatty v0.0.19
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
+	github.com/pkg/errors v0.9.1
 	github.com/rivo/tview v0.0.0-20230530133550-8bd761dda819
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
+	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3
 	golang.org/x/sys v0.8.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	go.opencensus.io v0.22.5 // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 )