Merge remote-tracking branch 'origin/master' into fix/perf-pebble-imp…

…rovements

core/commands/files.go

@@ -59,16 +59,18 @@ Content added with "ipfs add" (which by default also becomes pinned), is not
 added to MFS. Any content can be lazily referenced from MFS with the command
 "ipfs files cp /ipfs/<cid> /some/path/" (see ipfs files cp --help).
 
-
-NOTE:
-Most of the subcommands of 'ipfs files' accept the '--flush' flag. It defaults
-to true. Use caution when setting this flag to false. It will improve
+NOTE: Most of the subcommands of 'ipfs files' accept the '--flush' flag. It
+defaults to true and ensures two things: 1) that the changes are reflected in
+the full MFS structure (updated CIDs) 2) that the parent-folder's cache is
+cleared. Use caution when setting this flag to false. It will improve
 performance for large numbers of file operations, but it does so at the cost
-of consistency guarantees. If the daemon is unexpectedly killed before running
-'ipfs files flush' on the files in question, then data may be lost. This also
-applies to run 'ipfs repo gc' concurrently with '--flush=false'
-operations.
-`,
+of consistency guarantees and unbound growth of the directories' in-memory
+caches.  If the daemon is unexpectedly killed before running 'ipfs files
+flush' on the files in question, then data may be lost. This also applies to
+run 'ipfs repo gc' concurrently with '--flush=false' operations. We recommend
+flushing paths reguarly with 'ipfs files flush', specially the folders on
+which many write operations are happening, as a way to clear the directory
+cache, free memory and speed up read operations.`,
 	},
 	Options: []cmds.Option{
 		cmds.BoolOption(filesFlushOptionName, "f", "Flush target and ancestors after write.").WithDefault(true),
@@ -491,10 +493,14 @@ being GC'ed.
 		}
 
 		if flush {
-			_, err := mfs.FlushPath(req.Context, nd.FilesRoot, dst)
-			if err != nil {
+			if _, err := mfs.FlushPath(req.Context, nd.FilesRoot, dst); err != nil {
 				return fmt.Errorf("cp: cannot flush the created file %s: %s", dst, err)
 			}
+			// Flush parent to clear directory cache and free memory.
+			parent := gopath.Dir(dst)
+			if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parent); err != nil {
+				return fmt.Errorf("cp: cannot flush the created file's parent folder %s: %s", dst, err)
+			}
 		}
 
 		return nil
@@ -792,10 +798,30 @@ Example:
 		}
 
 		err = mfs.Mv(nd.FilesRoot, src, dst)
-		if err == nil && flush {
-			_, err = mfs.FlushPath(req.Context, nd.FilesRoot, "/")
+		if err != nil {
+			return err
 		}
-		return err
+		if flush {
+			parentSrc := gopath.Dir(src)
+			parentDst := gopath.Dir(dst)
+			// Flush parent to clear directory cache and free memory.
+			if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parentDst); err != nil {
+				return fmt.Errorf("cp: cannot flush the destination file's parent folder %s: %s", dst, err)
+			}
+
+			// Avoid re-flushing when moving within the same folder.
+			if parentSrc != parentDst {
+				if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parentSrc); err != nil {
+					return fmt.Errorf("cp: cannot flush the source's file's parent folder %s: %s", dst, err)
+				}
+			}
+
+			if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, "/"); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	},
 }
 
@@ -943,6 +969,17 @@ See '--to-files' in 'ipfs add --help' for more information.
 					flog.Error("files: error closing file mfs file descriptor", err)
 				}
 			}
+			if flush {
+				// Flush parent to clear directory cache and free memory.
+				parent := gopath.Dir(path)
+				if _, err := mfs.FlushPath(req.Context, nd.FilesRoot, parent); err != nil {
+					if retErr == nil {
+						retErr = err
+					} else {
+						flog.Error("files: flushing the parent folder", err)
+					}
+				}
+			}
 		}()
 
 		if trunc {
@@ -1105,11 +1142,20 @@ Change the CID version or hash function of the root node of a given path.
 			return err
 		}
 
-		err = updatePath(nd.FilesRoot, path, prefix)
-		if err == nil && flush {
-			_, err = mfs.FlushPath(req.Context, nd.FilesRoot, path)
+		if err := updatePath(nd.FilesRoot, path, prefix); err != nil {
+			return err
 		}
-		return err
+		if flush {
+			if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, path); err != nil {
+				return err
+			}
+			// Flush parent to clear directory cache and free memory.
+			parent := gopath.Dir(path)
+			if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parent); err != nil {
+				return err
+			}
+		}
+		return nil
 	},
 }
 

docs/changelogs/v0.33.md

@@ -9,6 +9,8 @@
   - [Bitswap improvements from Boxo](#bitswap-improvements-from-boxo)
   - [Using default `libp2p_rcmgr`  metrics](#using-default-libp2p_rcmgr--metrics)
   - [`ipfs add --to-files` no longer works with `--wrap`](#ipfs-add---to-files-no-longer-works-with---wrap)
+  - [New options for faster writes: `WriteThrough`, `BlockKeyCacheSize`, `BatchMaxNodes`, `BatchMaxSize`](#new-options-for-faster-writes-writethrough-blockkeycachesize-batchmaxnodes-batchmaxsize)
+  - [MFS stability with large number of writes](#mfs-stability-with-large-number-of-writes)
   - [📦️ Dependency updates](#-dependency-updates)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
@@ -49,11 +51,14 @@ As a reminder, details from all the options are explained in the [configuration
 
 We recommend users trying Pebble as a datastore backend to disable both blockstore bloom-filter and key caching layers and enable write through as a way to evaluate the raw performance of the underlying datastore, which includes its own bloom-filter and caching layers (default cache size is `8MiB` and can be configured in the [options](../datastores.md#pebbleds).
 
+#### MFS stability with large number of writes
+
+We have fixed a number of issues that were triggered by writing or copying many files onto an MFS folder: increased memory usage first, then CPU, disk usage, and eventually a deadlock on write operations. The details of the fixes can be read at [#10630](https://github.com/ipfs/kubo/pull/10630) and [#10623](https://github.com/ipfs/kubo/pull/10623). The result is that writing large amounts of files to an MFS folder should now be possible without major issues. It is possible, as before, to speed up the operations using the `ipfs files --flush=false <op> ...` flag, but it is recommended to switch to `ipfs files --flush=true <op> ...` regularly, or call `ipfs files flush` on the working directory regularly, as this will flush, clear the directory cache and speed up reads. 
 
 #### 📦️ Dependency updates
 
-- update `boxo` to [v0.25.0](https://github.com/ipfs/boxo/releases/tag/v0.25.0)
-- update `go-libp2p` to [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2)
+- update `boxo` to [v0.26.0](https://github.com/ipfs/boxo/releases/tag/v0.26.0)
+- update `go-libp2p` to [v0.38.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.38.1) (incl. [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2) + [v0.38.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.38.0))
 - update `p2p-forge/client` to [v0.1.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.1.0)
 - update `ipfs-webui` to [v4.4.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.1)