Skip to content

Commit

Permalink
Merge cockroachdb#132870
Browse files Browse the repository at this point in the history 
132870: storage: add cluster setting delete_compaction_excise.enabled r=aadityasondhi a=itsbilal

This change adds a cluster setting to be able to dynamically
disable excises in delete-only compactions. Excises allow
for range deletions / range key deletions to be applied to
sstables even if they don't fully overlap with the sstable;
we just produce virtual sstables for the remainder of the sstable.

Epic: none

Release note: None

Co-authored-by: Bilal Akhtar <[email protected]>
  • Loading branch information
@itsbilal
craig[bot] and itsbilal committed Oct 18, 2024
2 parents 9c3acd8 + 75fb9ef commit 833dadd
Show file tree
Hide file tree
Showing 7 changed files with 24 additions and 7 deletions.
6 changes: 3 additions & 3 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1818,10 +1818,10 @@ def go_deps():
patches = [
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
],
sha256 = "7136fa698f76055d99e3cf1055026d3fc00fc0cd6c65f7c27ef202ca769ea10f",
strip_prefix = "github.com/cockroachdb/[email protected]20241017141447-c6b097e7e6a0",
sha256 = "8c165990dc3d4d67618b19e45e2c79f5f48ab9df4e19f881ee1cfa82cdd009df",
strip_prefix = "github.com/cockroachdb/[email protected]20241017195839-1d2e9e829b92",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017141447-c6b097e7e6a0.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017195839-1d2e9e829b92.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion build/bazelutil/distdir_files.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20230118201751-21c54148d20b.zip": "ca7776f47e5fecb4c495490a679036bfc29d95bd7625290cfdb9abb0baf97476",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017141447-c6b097e7e6a0.zip": "7136fa698f76055d99e3cf1055026d3fc00fc0cd6c65f7c27ef202ca769ea10f",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017195839-1d2e9e829b92.zip": "8c165990dc3d4d67618b19e45e2c79f5f48ab9df4e19f881ee1cfa82cdd009df",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.5.zip": "11b30528eb0dafc8bc1a5ba39d81277c257cbe6946a7564402f588357c164560",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",
Expand Down
1 change: 1 addition & 0 deletions docs/generated/settings/settings-for-tenants.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,7 @@ sql.txn.read_committed_isolation.enabled boolean true set to true to allow trans
sql.txn.repeatable_read_isolation.enabled (alias: sql.txn.snapshot_isolation.enabled) boolean false set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands application
sql.txn_fingerprint_id_cache.capacity integer 100 the maximum number of txn fingerprint IDs stored application
storage.columnar_blocks.enabled boolean false set to true to enable columnar-blocks to store KVs in a columnar format system-visible
storage.delete_compaction_excise.enabled boolean true set to false to direct Pebble to not partially excise sstables in delete-only compactions system-visible
storage.ingestion.value_blocks.enabled boolean true set to true to enable writing of value blocks in ingestion sstables application
storage.max_sync_duration duration 20s maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash system-visible
storage.max_sync_duration.fatal.enabled boolean true if true, fatal the process when a disk operation exceeds storage.max_sync_duration application
Expand Down
1 change: 1 addition & 0 deletions docs/generated/settings/settings.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,7 @@
<tr><td><div id="setting-sql-txn-snapshot-isolation-enabled" class="anchored"><code>sql.txn.repeatable_read_isolation.enabled<br />(alias: sql.txn.snapshot_isolation.enabled)</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
<tr><td><div id="setting-sql-txn-fingerprint-id-cache-capacity" class="anchored"><code>sql.txn_fingerprint_id_cache.capacity</code></div></td><td>integer</td><td><code>100</code></td><td>the maximum number of txn fingerprint IDs stored</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
<tr><td><div id="setting-storage-columnar-blocks-enabled" class="anchored"><code>storage.columnar_blocks.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to enable columnar-blocks to store KVs in a columnar format</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
<tr><td><div id="setting-storage-delete-compaction-excise-enabled" class="anchored"><code>storage.delete_compaction_excise.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to direct Pebble to not partially excise sstables in delete-only compactions</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
<tr><td><div id="setting-storage-experimental-eventually-file-only-snapshots-enabled" class="anchored"><code>storage.experimental.eventually_file_only_snapshots.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to disable eventually-file-only-snapshots (kv.snapshot_receiver.excise.enabled must also be false)</td><td>Dedicated/Self-Hosted</td></tr>
<tr><td><div id="setting-storage-ingest-split-enabled" class="anchored"><code>storage.ingest_split.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to disable ingest-time splitting that lowers write-amplification</td><td>Dedicated/Self-Hosted</td></tr>
<tr><td><div id="setting-storage-ingestion-value-blocks-enabled" class="anchored"><code>storage.ingestion.value_blocks.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable writing of value blocks in ingestion sstables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ require (
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
github.com/cockroachdb/gostdlib v1.19.0
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b
github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0
github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92
github.com/cockroachdb/redact v1.1.5
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,8 +536,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0 h1:bxW1ydleLzHAb6u55RvwOgeOfrhXOF9iaGQ3cjWzvtM=
github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0/go.mod h1:XmS8uVDd9YFw/1R7J0J/CmTUANwT7iGnBRxH9AyDA90=
github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92 h1:AEWpYdO8k0gpPWZtpP8CyTr901vv7yxKVrzkXz5Vte8=
github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92/go.mod h1:XmS8uVDd9YFw/1R7J0J/CmTUANwT7iGnBRxH9AyDA90=
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
Expand Down
15 changes: 15 additions & 0 deletions pkg/storage/pebble.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,18 @@ var columnarBlocksEnabled = settings.RegisterBoolSetting(
settings.WithPublic,
)

// deleteCompactionsCanExcise controls whether delete compactions can
// apply rangedels/rangekeydels on sstables they partially apply to, through
// an excise operation, instead of just applying the rangedels/rangekeydels
// that fully delete sstables.
var deleteCompactionsCanExcise = settings.RegisterBoolSetting(
settings.SystemVisible,
"storage.delete_compaction_excise.enabled",
"set to false to direct Pebble to not partially excise sstables in delete-only compactions",
metamorphic.ConstantWithTestBool(
"storage.delete_compaction_excise.enabled", true), /* defaultValue */
settings.WithPublic)

// IngestAsFlushable controls whether ingested sstables that overlap the
// memtable may be lazily ingested: written to the WAL and enqueued in the list
// of flushables (eg, memtables, large batches and now lazily-ingested
Expand Down Expand Up @@ -1228,6 +1240,9 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
cfg.opts.Experimental.EnableColumnarBlocks = func() bool {
return columnarBlocksEnabled.Get(&cfg.settings.SV)
}
cfg.opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
return deleteCompactionsCanExcise.Get(&cfg.settings.SV)
}

auxDir := cfg.opts.FS.PathJoin(cfg.env.Dir, base.AuxiliaryDir)
if !cfg.env.IsReadOnly() {
Expand Down

0 comments on commit 833dadd

Please sign in to comment.