diff --git a/Makefile b/Makefile index 2ae9db11d..2b86437d8 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,9 @@ coverage-html: coverage gobench: go test -bench=. $(PACKAGE)/pkg/analyze +heap-profile: + go tool pprof -web http://localhost:6060/debug/pprof/heap + benchmark: sudo cpupower frequency-set -g performance hyperfine --export-markdown=bench-cold.md \ @@ -133,5 +136,6 @@ release: install-dev-dependencies: go install gotest.tools/gotestsum@latest + go install github.com/mitchellh/gox@latest .PHONY: run build build-static build-all test gobench benchmark coverage coverage-html clean clean-uncompressed-dist man show-man release diff --git a/README.md b/README.md index 9f626fde3..4acb36ae9 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Using curl: winget install gdu -You can either run it as `gdu_windows_amd64.exe` or +You can either run it as `gdu_windows_amd64.exe` or * add an alias with `Doskey`. * add `alias gdu="gdu_windows_amd64.exe"` to your `~/.bashrc` file if using Git Bash to run it as `gdu`. @@ -181,11 +181,14 @@ Flags: -p, --no-progress Do not show progress in non-interactive mode -n, --non-interactive Do not run in interactive mode -o, --output-file string Export all info into file as JSON + -r, --read-from-storage Read analysis data from persistent key-value storage -a, --show-apparent-size Show apparent size -d, --show-disks Show all mounted disks -B, --show-relative-size Show relative size --si Show sizes with decimal SI prefixes (kB, MB, GB) instead of binary prefixes (KiB, MiB, GiB) + --storage-path string Path to persistent key-value storage directory (default is /tmp/badger) (default "/tmp/badger") -s, --summarize Show only a total in non-interactive mode + --use-storage Use persistent key-value storage for analysis data (experimental) -v, --version Print version --write-config Write current configuration to file (default is $HOME/.gdu.yaml) @@ -221,6 +224,9 @@ In interactive mode: gdu -o- / | gzip -c >report.json.gz # write all info to JSON file for later analysis zcat report.json.gz | gdu -f- # read analysis from file + GOGC=10 gdu -g --use-storage / # use persistent key-value storage for saving analysis data + gdu -r / # read saved analysis data from persistent key-value storage + ## Modes Gdu has three modes: interactive (default), non-interactive and export. @@ -320,6 +326,18 @@ Example running gdu with constant GC, but not so aggressive as default: GOGC=200 gdu -g / ``` +## Saving analysis data to persistent key-value storage (experimental) + +Gdu can store the analysis data to persistent key-value storage instead of just memory. +Gdu will run much slower (approx 10x) but it should use much less memory (when using small GOGC as well). +Gdu can also reopen with the saved data. +Currently only BadgerDB is supported as the key-value storage (embedded). + +``` +GOGC=10 gdu -g --use-storage / # saves analysis data to key-value storage +gdu -r / # reads just saved data, does not run analysis again +``` + ## Running tests make install-dev-dependencies diff --git a/cmd/gdu/app/app.go b/cmd/gdu/app/app.go index 61973500e..472cecb09 100644 --- a/cmd/gdu/app/app.go +++ b/cmd/gdu/app/app.go @@ -16,6 +16,7 @@ import ( "github.com/dundee/gdu/v5/build" "github.com/dundee/gdu/v5/internal/common" + "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" gfs "github.com/dundee/gdu/v5/pkg/fs" "github.com/dundee/gdu/v5/report" @@ -30,11 +31,13 @@ type UI interface { ListDevices(getter device.DevicesInfoGetter) error AnalyzePath(path string, parentDir gfs.Item) error ReadAnalysis(input io.Reader) error + ReadFromStorage(storagePath, path string) error SetIgnoreDirPaths(paths []string) SetIgnoreDirPatterns(paths []string) error SetIgnoreFromFile(ignoreFile string) error SetIgnoreHidden(value bool) SetFollowSymlinks(value bool) + SetAnalyzer(analyzer common.Analyzer) StartUILoop() error } @@ -61,6 +64,9 @@ type Flags struct { FollowSymlinks bool `yaml:"follow-symlinks"` Profiling bool `yaml:"profiling"` ConstGC bool `yaml:"const-gc"` + UseStorage bool `yaml:"use-storage"` + StoragePath string `yaml:"storage-path"` + ReadFromStorage bool `yaml:"read-from-storage"` Summarize bool `yaml:"summarize"` UseSIPrefix bool `yaml:"use-si-prefix"` NoPrefix bool `yaml:"no-prefix"` @@ -135,6 +141,12 @@ func (a *App) Run() (err error) { return } + if a.Flags.UseStorage { + ui.SetAnalyzer(analyze.CreateStoredAnalyzer(a.Flags.StoragePath)) + } + if a.Flags.FollowSymlinks { + ui.SetFollowSymlinks(true) + } if err = a.setNoCross(path); err != nil { return } @@ -273,10 +285,6 @@ func (a *App) createUI() (UI, error) { tview.Styles.BorderColor = tcell.ColorDefault } - if a.Flags.FollowSymlinks { - ui.SetFollowSymlinks(true) - } - return ui, nil } @@ -324,6 +332,11 @@ func (a *App) runAction(ui UI, path string) error { if err := ui.ReadAnalysis(input); err != nil { return fmt.Errorf("reading analysis: %w", err) } + } else if a.Flags.ReadFromStorage { + ui.SetAnalyzer(analyze.CreateStoredAnalyzer(a.Flags.StoragePath)) + if err := ui.ReadFromStorage(a.Flags.StoragePath, path); err != nil { + return fmt.Errorf("reading from storage (%s): %w", a.Flags.StoragePath, err) + } } else { if build.RootPathPrefix != "" { path = build.RootPathPrefix + path diff --git a/cmd/gdu/main.go b/cmd/gdu/main.go index 0c28296c5..04bca33e9 100644 --- a/cmd/gdu/main.go +++ b/cmd/gdu/main.go @@ -57,6 +57,10 @@ func init() { flags.BoolVarP(&af.ConstGC, "const-gc", "g", false, "Enable memory garbage collection during analysis with constant level set by GOGC") flags.BoolVar(&af.Profiling, "enable-profiling", false, "Enable collection of profiling data and provide it on http://localhost:6060/debug/pprof/") + flags.BoolVar(&af.UseStorage, "use-storage", false, "Use persistent key-value storage for analysis data (experimental)") + flags.StringVar(&af.StoragePath, "storage-path", "/tmp/badger", "Path to persistent key-value storage directory (default is /tmp/badger)") + flags.BoolVarP(&af.ReadFromStorage, "read-from-storage", "r", false, "Read analysis data from persistent key-value storage") + flags.BoolVarP(&af.ShowDisks, "show-disks", "d", false, "Show all mounted disks") flags.BoolVarP(&af.ShowApparentSize, "show-apparent-size", "a", false, "Show apparent size") flags.BoolVarP(&af.ShowRelativeSize, "show-relative-size", "B", false, "Show relative size") diff --git a/go.mod b/go.mod index 4964df70e..c1aad7988 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,39 @@ module github.com/dundee/gdu/v5 -go 1.18 +go 1.20 require ( + github.com/dgraph-io/badger/v3 v3.2103.2 github.com/fatih/color v1.15.0 github.com/gdamore/tcell/v2 v2.6.0 github.com/maruel/natural v1.1.0 github.com/mattn/go-isatty v0.0.19 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/pkg/errors v0.9.1 github.com/rivo/tview v0.0.0-20230530133550-8bd761dda819 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 + golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 golang.org/x/sys v0.8.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.3.1 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.12.3 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -27,7 +41,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/spf13/pflag v1.0.5 // indirect + go.opencensus.io v0.22.5 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index 37dea1e01..801f7885a 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,61 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= +github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.6.0 h1:OKbluoP9VYmJwZwq/iLb4BxwKcwGthaa1YNBJIyCySg= github.com/gdamore/tcell/v2 v2.6.0/go.mod h1:be9omFATkdr0D9qewWW3d+MEvl5dha+Etb5y65J2H8Y= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -20,6 +65,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -29,8 +75,13 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/tview v0.0.0-20230530133550-8bd761dda819 h1:qRMCGgwKl66uWe7Hnzl5bCvZlfrLNIxOx7K00j5XeNc= @@ -41,28 +92,77 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -84,14 +184,30 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/common/ui.go b/internal/common/ui.go index 259312c1d..2caefd3b1 100644 --- a/internal/common/ui.go +++ b/internal/common/ui.go @@ -19,6 +19,11 @@ type UI struct { ConstGC bool } +// SetAnalyzer sets analyzer instance +func (ui *UI) SetAnalyzer(a Analyzer) { + ui.Analyzer = a +} + // SetFollowSymlinks sets whether symlinks to files should be followed func (ui *UI) SetFollowSymlinks(v bool) { ui.Analyzer.SetFollowSymlinks(v) diff --git a/pkg/analyze/file.go b/pkg/analyze/file.go index b3ec6b50a..522d0a8a0 100644 --- a/pkg/analyze/file.go +++ b/pkg/analyze/file.go @@ -122,6 +122,11 @@ func (f *File) AddFile(item fs.Item) { panic("AddFile should not be called on file") } +// RemoveFile panics on file +func (f *File) RemoveFile(item fs.Item) { + panic("RemoveFile should not be called on file") +} + // Dir struct type Dir struct { *File @@ -182,7 +187,7 @@ func (f *Dir) UpdateStats(linkedItems fs.HardLinkedItems) { totalSize := int64(4096) totalUsage := int64(4096) var itemCount int - for _, entry := range f.Files { + for _, entry := range f.GetFiles() { count, size, usage := entry.GetItemStats(linkedItems) totalSize += size totalUsage += usage @@ -204,16 +209,11 @@ func (f *Dir) UpdateStats(linkedItems fs.HardLinkedItems) { f.Usage = totalUsage } -// RemoveItemFromDir removes item from dir -func RemoveItemFromDir(dir fs.Item, item fs.Item) error { - err := os.RemoveAll(item.GetPath()) - if err != nil { - return err - } - - dir.SetFiles(dir.GetFiles().Remove(item)) +// RemoveFile panics on file +func (f *Dir) RemoveFile(item fs.Item) { + f.SetFiles(f.GetFiles().Remove(item)) - cur := dir.(*Dir) + cur := f for { cur.ItemCount -= item.GetItemCount() cur.Size -= item.GetSize() @@ -224,6 +224,16 @@ func RemoveItemFromDir(dir fs.Item, item fs.Item) error { } cur = cur.Parent.(*Dir) } +} + +// RemoveItemFromDir removes item from dir +func RemoveItemFromDir(dir fs.Item, item fs.Item) error { + err := os.RemoveAll(item.GetPath()) + if err != nil { + return err + } + + dir.RemoveFile(item) return nil } diff --git a/pkg/analyze/storage.go b/pkg/analyze/storage.go new file mode 100644 index 000000000..99c2958d6 --- /dev/null +++ b/pkg/analyze/storage.go @@ -0,0 +1,141 @@ +package analyze + +import ( + "bytes" + "encoding/gob" + "path/filepath" + "sync" + + "github.com/dgraph-io/badger/v3" + "github.com/dundee/gdu/v5/pkg/fs" + "github.com/pkg/errors" +) + +func init() { + gob.RegisterName("analyze.StoredDir", &StoredDir{}) + gob.RegisterName("analyze.Dir", &Dir{}) + gob.RegisterName("analyze.File", &File{}) + gob.RegisterName("analyze.ParentDir", &ParentDir{}) +} + +// DefaultStorage is a default instance of badger storage +var DefaultStorage *Storage + +// Storage represents a badger storage +type Storage struct { + db *badger.DB + storagePath string + topDir string + m sync.RWMutex + counter int + counterM sync.Mutex +} + +// NewStorage returns new instance of badger storage +func NewStorage(storagePath, topDir string) *Storage { + st := &Storage{ + storagePath: storagePath, + topDir: topDir, + } + DefaultStorage = st + return st +} + +// GetTopDir returns top directory +func (s *Storage) GetTopDir() string { + return s.topDir +} + +// IsOpen returns true if badger DB is open +func (s *Storage) IsOpen() bool { + s.m.RLock() + defer s.m.RUnlock() + return s.db != nil +} + +// Open opens badger DB +func (s *Storage) Open() func() { + options := badger.DefaultOptions(s.storagePath) + options.Logger = nil + db, err := badger.Open(options) + if err != nil { + panic(err) + } + s.db = db + + return func() { + s.db.Close() + s.db = nil + } +} + +// StoreDir saves item info into badger DB +func (s *Storage) StoreDir(dir fs.Item) error { + s.checkCount() + s.m.RLock() + defer s.m.RUnlock() + + return s.db.Update(func(txn *badger.Txn) error { + b := &bytes.Buffer{} + enc := gob.NewEncoder(b) + err := enc.Encode(dir) + if err != nil { + return errors.Wrap(err, "encoding dir value") + } + + return txn.Set([]byte(dir.GetPath()), b.Bytes()) + }) +} + +// LoadDir saves item info into badger DB +func (s *Storage) LoadDir(dir fs.Item) error { + s.checkCount() + s.m.RLock() + defer s.m.RUnlock() + + return s.db.View(func(txn *badger.Txn) error { + path := dir.GetPath() + item, err := txn.Get([]byte(path)) + if err != nil { + return errors.Wrap(err, "reading stored value for path: "+path) + } + return item.Value(func(val []byte) error { + b := bytes.NewBuffer(val) + dec := gob.NewDecoder(b) + return dec.Decode(dir) + }) + }) +} + +// GetDirForPath returns Dir for given path +func (s *Storage) GetDirForPath(path string) (fs.Item, error) { + dirPath := filepath.Dir(path) + name := filepath.Base(path) + dir := &StoredDir{ + &Dir{ + File: &File{ + Name: name, + }, + BasePath: dirPath, + }, + nil, + } + err := s.LoadDir(dir) + if err != nil { + return nil, err + } + return dir, nil +} + +func (s *Storage) checkCount() { + s.counterM.Lock() + defer s.counterM.Unlock() + s.counter++ + if s.counter >= 10000 { + s.m.Lock() + defer s.m.Unlock() + s.counter = 0 + s.db.Close() + s.Open() + } +} diff --git a/pkg/analyze/stored.go b/pkg/analyze/stored.go new file mode 100644 index 000000000..14182c5ca --- /dev/null +++ b/pkg/analyze/stored.go @@ -0,0 +1,382 @@ +package analyze + +import ( + "io" + "os" + "path/filepath" + "runtime/debug" + "time" + + "github.com/dundee/gdu/v5/internal/common" + "github.com/dundee/gdu/v5/pkg/fs" + log "github.com/sirupsen/logrus" +) + +// StoredAnalyzer implements Analyzer +type StoredAnalyzer struct { + storage *Storage + storagePath string + progress *common.CurrentProgress + progressChan chan common.CurrentProgress + progressOutChan chan common.CurrentProgress + progressDoneChan chan struct{} + doneChan common.SignalGroup + wait *WaitGroup + ignoreDir common.ShouldDirBeIgnored + followSymlinks bool +} + +// CreateStoredAnalyzer returns Analyzer +func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer { + return &StoredAnalyzer{ + storagePath: storagePath, + progress: &common.CurrentProgress{ + ItemCount: 0, + TotalSize: int64(0), + }, + progressChan: make(chan common.CurrentProgress, 1), + progressOutChan: make(chan common.CurrentProgress, 1), + progressDoneChan: make(chan struct{}), + doneChan: make(common.SignalGroup), + wait: (&WaitGroup{}).Init(), + } +} + +// GetProgressChan returns channel for getting progress +func (a *StoredAnalyzer) GetProgressChan() chan common.CurrentProgress { + return a.progressOutChan +} + +// GetDone returns channel for checking when analysis is done +func (a *StoredAnalyzer) GetDone() common.SignalGroup { + return a.doneChan +} + +func (a *StoredAnalyzer) SetFollowSymlinks(v bool) { + a.followSymlinks = v +} + +// ResetProgress returns progress +func (a *StoredAnalyzer) ResetProgress() { + a.progress = &common.CurrentProgress{} + a.progressChan = make(chan common.CurrentProgress, 1) + a.progressOutChan = make(chan common.CurrentProgress, 1) + a.progressDoneChan = make(chan struct{}) + a.doneChan = make(common.SignalGroup) + a.wait = (&WaitGroup{}).Init() +} + +// AnalyzeDir analyzes given path +func (a *StoredAnalyzer) AnalyzeDir( + path string, ignore common.ShouldDirBeIgnored, constGC bool, +) fs.Item { + if !constGC { + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + go manageMemoryUsage(a.doneChan) + } + + a.storage = NewStorage(a.storagePath, path) + closeFn := a.storage.Open() + defer func() { + // nasty hack to close storage after all goroutines are done + // Wait returns immediately if value is 0 + // few last goroutines might still start after that + time.Sleep(1 * time.Second) + closeFn() + }() + + a.ignoreDir = ignore + + go a.updateProgress() + dir := a.processDir(path) + + a.wait.Wait() + + a.progressDoneChan <- struct{}{} + a.doneChan.Broadcast() + + return dir +} + +func (a *StoredAnalyzer) processDir(path string) *StoredDir { + var ( + file *File + err error + totalSize int64 + info os.FileInfo + dirCount int + ) + + a.wait.Add(1) + + files, err := os.ReadDir(path) + if err != nil { + log.Print(err.Error()) + } + + dir := &StoredDir{ + Dir: &Dir{ + File: &File{ + Name: filepath.Base(path), + Flag: getDirFlag(err, len(files)), + }, + BasePath: filepath.Dir(path), + ItemCount: 1, + Files: make(fs.Files, 0, len(files)), + }, + } + parent := &ParentDir{Path: path} + + setDirPlatformSpecificAttrs(dir.Dir, path) + + for _, f := range files { + name := f.Name() + entryPath := filepath.Join(path, name) + if f.IsDir() { + if a.ignoreDir(name, entryPath) { + continue + } + dirCount++ + + subdir := &StoredDir{ + &Dir{ + File: &File{ + Name: name, + }, + BasePath: path, + }, + nil, + } + dir.AddFile(subdir) + + go func(entryPath string) { + concurrencyLimit <- struct{}{} + a.processDir(entryPath) + <-concurrencyLimit + }(entryPath) + } else { + info, err = f.Info() + if err != nil { + log.Print(err.Error()) + continue + } + file = &File{ + Name: name, + Flag: getFlag(info), + Size: info.Size(), + Parent: parent, + } + setPlatformSpecificAttrs(file, info) + + totalSize += info.Size() + + dir.AddFile(file) + } + } + + err = a.storage.StoreDir(dir) + if err != nil { + log.Print(err.Error()) + } + + a.wait.Done() + + a.progressChan <- common.CurrentProgress{ + CurrentItemName: path, + ItemCount: len(files), + TotalSize: totalSize, + } + return dir +} + +func (a *StoredAnalyzer) updateProgress() { + for { + select { + case <-a.progressDoneChan: + return + case progress := <-a.progressChan: + a.progress.CurrentItemName = progress.CurrentItemName + a.progress.ItemCount += progress.ItemCount + a.progress.TotalSize += progress.TotalSize + } + + select { + case a.progressOutChan <- *a.progress: + default: + } + } +} + +// StoredDir implements Dir item stored on disk +type StoredDir struct { + *Dir + cachedFiles fs.Files +} + +// GetParent returns parent dir +func (f *StoredDir) GetParent() fs.Item { + if DefaultStorage.GetTopDir() == f.GetPath() { + return nil + } + + if !DefaultStorage.IsOpen() { + closeFn := DefaultStorage.Open() + defer closeFn() + } + + dir, err := DefaultStorage.GetDirForPath(f.BasePath) + if err != nil { + log.Print(err.Error()) + } + return dir +} + +// GetFiles returns files in directory +// If files are already cached, return them +// Otherwise load them from storage +func (f *StoredDir) GetFiles() fs.Files { + if f.cachedFiles != nil { + return f.cachedFiles + } + + if !DefaultStorage.IsOpen() { + closeFn := DefaultStorage.Open() + defer closeFn() + } + + var files fs.Files + for _, file := range f.Files { + if file.IsDir() { + dir := &StoredDir{ + &Dir{ + File: &File{ + Name: file.GetName(), + }, + BasePath: f.GetPath(), + }, + nil, + } + + err := DefaultStorage.LoadDir(dir) + if err != nil { + log.Print(err.Error()) + } + files = append(files, dir) + } else { + files = append(files, file) + } + } + + f.cachedFiles = files + return files +} + +// SetFiles sets files in directory +func (f *StoredDir) SetFiles(files fs.Files) { + f.Files = files +} + +// RemoveFile panics on file +func (f *StoredDir) RemoveFile(item fs.Item) { + if !DefaultStorage.IsOpen() { + closeFn := DefaultStorage.Open() + defer closeFn() + } + + f.SetFiles(f.GetFiles().Remove(item)) + f.cachedFiles = nil + + cur := f + for { + cur.ItemCount -= item.GetItemCount() + cur.Size -= item.GetSize() + cur.Usage -= item.GetUsage() + + err := DefaultStorage.StoreDir(cur) + if err != nil { + log.Print(err.Error()) + } + + parent := cur.GetParent() + if parent == nil { + break + } + cur = parent.(*StoredDir) + } +} + +// GetItemStats returns item count, apparent usage and real usage of this dir +func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems) (int, int64, int64) { + f.UpdateStats(linkedItems) + return f.ItemCount, f.GetSize(), f.GetUsage() +} + +// UpdateStats recursively updates size and item count +func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) { + if !DefaultStorage.IsOpen() { + closeFn := DefaultStorage.Open() + defer closeFn() + } + + totalSize := int64(4096) + totalUsage := int64(4096) + var itemCount int + f.cachedFiles = nil + for _, entry := range f.GetFiles() { + count, size, usage := entry.GetItemStats(linkedItems) + totalSize += size + totalUsage += usage + itemCount += count + + if entry.GetMtime().After(f.Mtime) { + f.Mtime = entry.GetMtime() + } + + switch entry.GetFlag() { + case '!', '.': + if f.Flag != '!' { + f.Flag = '.' + } + } + } + f.cachedFiles = nil + f.ItemCount = itemCount + 1 + f.Size = totalSize + f.Usage = totalUsage + err := DefaultStorage.StoreDir(f) + if err != nil { + log.Print(err.Error()) + } +} + +// ParentDir represents parent directory of single file +// It is used to get path to parent directory of a file +type ParentDir struct { + Path string +} + +func (p *ParentDir) GetPath() string { + return p.Path +} +func (p *ParentDir) GetName() string { panic("must not be called") } +func (p *ParentDir) GetFlag() rune { panic("must not be called") } +func (p *ParentDir) IsDir() bool { panic("must not be called") } +func (p *ParentDir) GetSize() int64 { panic("must not be called") } +func (p *ParentDir) GetType() string { panic("must not be called") } +func (p *ParentDir) GetUsage() int64 { panic("must not be called") } +func (p *ParentDir) GetMtime() time.Time { panic("must not be called") } +func (p *ParentDir) GetItemCount() int { panic("must not be called") } +func (p *ParentDir) GetParent() fs.Item { panic("must not be called") } +func (p *ParentDir) SetParent(fs.Item) { panic("must not be called") } +func (p *ParentDir) GetMultiLinkedInode() uint64 { panic("must not be called") } +func (p *ParentDir) EncodeJSON(writer io.Writer, topLevel bool) error { panic("must not be called") } +func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems) { panic("must not be called") } +func (p *ParentDir) AddFile(fs.Item) { panic("must not be called") } +func (p *ParentDir) GetFiles() fs.Files { panic("must not be called") } +func (p *ParentDir) SetFiles(fs.Files) { panic("must not be called") } +func (f *ParentDir) RemoveFile(item fs.Item) { panic("must not be called") } +func (p *ParentDir) GetItemStats( + linkedItems fs.HardLinkedItems, +) (int, int64, int64) { + panic("must not be called") +} diff --git a/pkg/analyze/stored_test.go b/pkg/analyze/stored_test.go new file mode 100644 index 000000000..bb8b52f2c --- /dev/null +++ b/pkg/analyze/stored_test.go @@ -0,0 +1,109 @@ +package analyze + +import ( + "bytes" + "encoding/gob" + "fmt" + "testing" + + "github.com/dundee/gdu/v5/internal/testdir" + "github.com/dundee/gdu/v5/pkg/fs" + "github.com/stretchr/testify/assert" +) + +func TestEncDec(t *testing.T) { + var d fs.Item = &StoredDir{ + Dir: &Dir{ + File: &File{ + Name: "xxx", + }, + BasePath: "/yyy", + }, + } + + b := &bytes.Buffer{} + enc := gob.NewEncoder(b) + err := enc.Encode(d) + assert.NoError(t, err) + + var x fs.Item = &StoredDir{} + dec := gob.NewDecoder(b) + err = dec.Decode(x) + assert.NoError(t, err) + + fmt.Println(d, x) + assert.Equal(t, d.GetName(), x.GetName()) +} + +func TestStoredAnalyzer(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + a := CreateStoredAnalyzer("/tmp/badger") + dir := a.AnalyzeDir( + "test_dir", func(_, _ string) bool { return false }, false, + ).(*StoredDir) + + a.GetDone().Wait() + + dir.UpdateStats(make(fs.HardLinkedItems)) + + // test dir info + assert.Equal(t, "test_dir", dir.Name) + assert.Equal(t, int64(7+4096*3), dir.Size) + assert.Equal(t, 5, dir.ItemCount) + assert.True(t, dir.IsDir()) + + // test dir tree + assert.Equal(t, "nested", dir.GetFiles()[0].GetName()) + assert.Equal(t, "subnested", dir.GetFiles()[0].(*StoredDir).GetFiles()[1].GetName()) + + // test file + assert.Equal(t, "file2", dir.GetFiles()[0].(*StoredDir).GetFiles()[0].GetName()) + assert.Equal(t, int64(2), dir.GetFiles()[0].(*StoredDir).GetFiles()[0].GetSize()) + assert.Equal(t, int64(4096), dir.GetFiles()[0].(*StoredDir).GetFiles()[0].GetUsage()) + + assert.Equal( + t, "file", dir.GetFiles()[0].(*StoredDir).GetFiles()[1].(*StoredDir).GetFiles()[0].GetName(), + ) + assert.Equal( + t, int64(5), dir.GetFiles()[0].(*StoredDir).GetFiles()[1].(*StoredDir).GetFiles()[0].GetSize(), + ) +} + +func TestRemoveStoredFile(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + a := CreateStoredAnalyzer("/tmp/badger") + dir := a.AnalyzeDir( + "test_dir", func(_, _ string) bool { return false }, false, + ).(*StoredDir) + + a.GetDone().Wait() + a.ResetProgress() + + dir.UpdateStats(make(fs.HardLinkedItems)) + + // test dir info + assert.Equal(t, "test_dir", dir.Name) + assert.Equal(t, int64(7+4096*3), dir.Size) + assert.Equal(t, 5, dir.ItemCount) + assert.True(t, dir.IsDir()) + + subdir := dir.GetFiles()[0].(*StoredDir) + subdir.RemoveFile(subdir.GetFiles()[0]) + + closeFn := DefaultStorage.Open() + defer closeFn() + stored, err := DefaultStorage.GetDirForPath("test_dir") + assert.NoError(t, err) + + assert.Equal(t, 4, stored.GetItemCount()) + assert.Equal(t, int64(5+4096*3), stored.GetSize()) + + file := stored.GetFiles()[0].GetFiles()[0].GetFiles()[0] + assert.Equal(t, false, file.IsDir()) + assert.Equal(t, "file", file.GetName()) + assert.Equal(t, "test_dir/nested/subnested", file.GetParent().GetPath()) +} diff --git a/pkg/analyze/wait.go b/pkg/analyze/wait.go index 3c4f63e1f..204921a54 100644 --- a/pkg/analyze/wait.go +++ b/pkg/analyze/wait.go @@ -43,6 +43,7 @@ func (s *WaitGroup) Wait() { func (s *WaitGroup) check() { if s.value == 0 { + s.wait.TryLock() s.wait.Unlock() } } diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 94d614c91..368ddaaf5 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -27,6 +27,7 @@ type Item interface { AddFile(Item) GetFiles() Files SetFiles(Files) + RemoveFile(Item) } // Files - slice of pointers to File diff --git a/report/export.go b/report/export.go index 3995262af..e0119339a 100644 --- a/report/export.go +++ b/report/export.go @@ -75,13 +75,35 @@ func (ui *UI) ReadAnalysis(input io.Reader) error { return errors.New("Reading analysis is not possible while exporting") } +// ReadFromStorage reads analysis data from persistent key-value storage +func (ui *UI) ReadFromStorage(storagePath, path string) error { + storage := analyze.NewStorage(storagePath, path) + closeFn := storage.Open() + defer closeFn() + + dir, err := storage.GetDirForPath(path) + if err != nil { + return err + } + + var waitWritten sync.WaitGroup + if ui.ShowProgress { + waitWritten.Add(1) + go func() { + defer waitWritten.Done() + ui.updateProgress() + }() + } + + return ui.exportDir(dir, &waitWritten) +} + // AnalyzePath analyzes recursively disk usage in given path func (ui *UI) AnalyzePath(path string, _ fs.Item) error { var ( dir fs.Item wait sync.WaitGroup waitWritten sync.WaitGroup - err error ) if ui.ShowProgress { @@ -101,9 +123,16 @@ func (ui *UI) AnalyzePath(path string, _ fs.Item) error { wait.Wait() + return ui.exportDir(dir, &waitWritten) +} + +func (ui *UI) exportDir(dir fs.Item, waitWritten *sync.WaitGroup) error { sort.Sort(sort.Reverse(dir.GetFiles())) - var buff bytes.Buffer + var ( + buff bytes.Buffer + err error + ) buff.Write([]byte(`[1,2,{"progname":"gdu","progver":"`)) buff.Write([]byte(build.Version)) diff --git a/stdout/stdout.go b/stdout/stdout.go index df47029d1..5b1ecba84 100644 --- a/stdout/stdout.go +++ b/stdout/stdout.go @@ -134,15 +134,17 @@ func (ui *UI) ListDevices(getter device.DevicesInfoGetter) error { // AnalyzePath analyzes recursively disk usage in given path func (ui *UI) AnalyzePath(path string, _ fs.Item) error { var ( - dir fs.Item - wait sync.WaitGroup + dir fs.Item + wait sync.WaitGroup + updateStatsDone chan struct{} ) + updateStatsDone = make(chan struct{}, 1) if ui.ShowProgress { wait.Add(1) go func() { defer wait.Done() - ui.updateProgress() + ui.updateProgress(updateStatsDone) }() } @@ -151,6 +153,7 @@ func (ui *UI) AnalyzePath(path string, _ fs.Item) error { defer wait.Done() dir = ui.Analyzer.AnalyzeDir(path, ui.CreateIgnoreFunc(), ui.ConstGC) dir.UpdateStats(make(fs.HardLinkedItems, 10)) + updateStatsDone <- struct{}{} }() wait.Wait() @@ -164,6 +167,25 @@ func (ui *UI) AnalyzePath(path string, _ fs.Item) error { return nil } +// ReadFromStorage reads analysis data from persistent key-value storage +func (ui *UI) ReadFromStorage(storagePath, path string) error { + storage := analyze.NewStorage(storagePath, path) + closeFn := storage.Open() + defer closeFn() + + dir, err := storage.GetDirForPath(path) + if err != nil { + return err + } + + if ui.summarize { + ui.printTotalItem(dir) + } else { + ui.showDir(dir) + } + return nil +} + func (ui *UI) showDir(dir fs.Item) { sort.Sort(sort.Reverse(dir.GetFiles())) @@ -303,14 +325,14 @@ func (ui *UI) showReadingProgress(doneChan chan struct{}) { } } -func (ui *UI) updateProgress() { +func (ui *UI) updateProgress(updateStatsDone <-chan struct{}) { emptyRow := "\r" for j := 0; j < 100; j++ { emptyRow += " " } progressChan := ui.Analyzer.GetProgressChan() - doneChan := ui.Analyzer.GetDone() + analysisDoneChan := ui.Analyzer.GetDone() var progress common.CurrentProgress @@ -320,9 +342,23 @@ func (ui *UI) updateProgress() { select { case progress = <-progressChan: - case <-doneChan: - fmt.Fprint(ui.output, "\r") - return + case <-analysisDoneChan: + for { + fmt.Fprint(ui.output, emptyRow) + fmt.Fprintf(ui.output, "\r %s ", string(progressRunes[i])) + fmt.Fprint(ui.output, "Calculating disk usage...") + time.Sleep(100 * time.Millisecond) + i++ + i %= 10 + + select { + case <-updateStatsDone: + fmt.Fprint(ui.output, emptyRow) + fmt.Fprint(ui.output, "\r") + return + default: + } + } } fmt.Fprintf(ui.output, "\r %s ", string(progressRunes[i])) diff --git a/tui/actions.go b/tui/actions.go index c9b39d603..a05e71ed0 100644 --- a/tui/actions.go +++ b/tui/actions.go @@ -136,6 +136,25 @@ func (ui *UI) ReadAnalysis(input io.Reader) error { return nil } +// ReadFromStorage reads analysis data from persistent key-value storage +func (ui *UI) ReadFromStorage(storagePath, path string) error { + storage := analyze.NewStorage(storagePath, path) + closeFn := storage.Open() + defer closeFn() + + dir, err := storage.GetDirForPath(path) + if err != nil { + return err + } + + ui.currentDir = dir + ui.topDirPath = ui.currentDir.GetPath() + ui.topDir = ui.currentDir + + ui.showDir() + return nil +} + func (ui *UI) delete(shouldEmpty bool) { if len(ui.markedRows) > 0 { ui.deleteMarked(shouldEmpty) diff --git a/tui/progress.go b/tui/progress.go index d21bb7571..e4293e17c 100644 --- a/tui/progress.go +++ b/tui/progress.go @@ -23,6 +23,10 @@ func (ui *UI) updateProgress() { select { case progress = <-progressChan: case <-doneChan: + ui.app.QueueUpdateDraw(func() { + ui.progress.SetTitle(" Finalizing... ") + ui.progress.SetText("Calculating disk usage...") + }) return } diff --git a/tui/tui.go b/tui/tui.go index baaee7348..129a1401d 100644 --- a/tui/tui.go +++ b/tui/tui.go @@ -3,6 +3,8 @@ package tui import ( "io" + "golang.org/x/exp/slices" + log "github.com/sirupsen/logrus" "github.com/dundee/gdu/v5/internal/common" @@ -222,14 +224,19 @@ func (ui *UI) fileItemSelected(row, column int) { return } - ui.currentDir = selectedDir.(*analyze.Dir) + ui.currentDir = selectedDir ui.hideFilterInput() ui.markedRows = make(map[int]struct{}) ui.showDir() - if selectedDir == origDir.GetParent() { - index, _ := ui.currentDir.GetFiles().IndexOf(origDir) - if ui.currentDir != ui.topDir { + if origDir.GetParent() != nil && selectedDir.GetName() == origDir.GetParent().GetName() { + index := slices.IndexFunc( + ui.currentDir.GetFiles(), + func(v fs.Item) bool { + return v.GetName() == origDir.GetName() + }, + ) + if ui.currentDir.GetPath() != ui.topDir.GetPath() { index++ } ui.table.Select(index, 0) diff --git a/tui/tui_test.go b/tui/tui_test.go index db2c5ce06..24b522ddf 100644 --- a/tui/tui_test.go +++ b/tui/tui_test.go @@ -67,9 +67,10 @@ func TestFooter(t *testing.T) { } func TestUpdateProgress(t *testing.T) { - app, simScreen := testapp.CreateTestAppWithSimScreen(15, 15) + simScreen := testapp.CreateSimScreen() defer simScreen.Fini() + app := testapp.CreateMockedApp(true) ui := CreateUI(app, simScreen, &bytes.Buffer{}, false, false, false, false, false) done := ui.Analyzer.GetDone() done.Broadcast()