From e9db14a0e8b026fd491372886cf7935e15015bde Mon Sep 17 00:00:00 2001 From: David Taylor Date: Fri, 29 Mar 2024 12:43:42 +0000 Subject: [PATCH] restore: waive 200 file limit for online restores Online restore wants big/huge chunks of spans passed to its workers so that they spend the most time splitting and linking inside a chunk and less time splitting chunks away from other workers. This is why we bump the target size to 16gb, but we also don't want a 200 file limit getting in the way of hitting this size. Release note: none. Epic: none. --- pkg/ccl/backupccl/restore_span_covering.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/ccl/backupccl/restore_span_covering.go b/pkg/ccl/backupccl/restore_span_covering.go index 4f1837126b5..a8c7e2c28ac 100644 --- a/pkg/ccl/backupccl/restore_span_covering.go +++ b/pkg/ccl/backupccl/restore_span_covering.go @@ -433,7 +433,11 @@ func generateAndSendImportSpans( // we want to bound sum total open files across all of them to <= 1k. // We bound the span byte size to improve work distribution and make // the progress more granular. - fits := lastCovSpanSize+newCovFilesSize <= filter.targetSize && lastCovSpanCount+newCovFilesCount <= 200 + // We waive the file count limit if the target size is >8GB, as a size + // this large clearly indicates we want lots of files (such as during + // online restores where we don't open them). + fileCountOK := lastCovSpanCount+newCovFilesCount <= 200 || filter.targetSize > 8<<30 + fits := lastCovSpanSize+newCovFilesSize <= filter.targetSize && fileCountOK if (newCovFilesCount == 0 || fits) && !firstInSpan { // If there are no new files that cover this span or if we can add the