From 696fe102ebebc3d6adc50670886a2f35ece1c97f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 7 Jun 2024 13:17:02 +0930 Subject: [PATCH 01/35] btrfs-progs: tests: do not use async discard for misc/004 [BUG] There is a long existing failure in my local VM that with any newer kernel (6.x) the test case misc/004 would fail with ENOSPC during balance: [TEST] misc-tests.sh [TEST/misc] 004-shrink-fs failed: /home/adam/btrfs-progs/btrfs balance start -mconvert=single -sconvert=single -f /home/adam/btrfs-progs/tests/mnt test failed for case 004-shrink-fs make: *** [Makefile:547: test-misc] Error 1 [CAUSE] With more testing, it turns out that just before the balance, the filesystem still have several empty data block groups. The reason is the new default discard=async behavior, as it also changes the empty block groups to be async, this leave the empty block groups there, resulting no extra space for the convert balance. [FIX] I do not understand why for loop block devices we also enable discard, but at least disable discard for the test case so that we can ensure the empty block groups get cleaned up properly. Pull-request: #809 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- tests/misc-tests/004-shrink-fs/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/misc-tests/004-shrink-fs/test.sh b/tests/misc-tests/004-shrink-fs/test.sh index c747364902..4fb42a024b 100755 --- a/tests/misc-tests/004-shrink-fs/test.sh +++ b/tests/misc-tests/004-shrink-fs/test.sh @@ -32,7 +32,9 @@ shrink_test() run_check truncate -s 20G "$IMAGE" run_check "$TOP/mkfs.btrfs" -f "$IMAGE" -run_check $SUDO_HELPER mount "$IMAGE" "$TEST_MNT" +# Disable the new default async discard, which makes empty block group cleanup +# async. +run_check $SUDO_HELPER mount -o nodiscard "$IMAGE" "$TEST_MNT" run_check $SUDO_HELPER chmod a+rw "$TEST_MNT" # Create 7 data block groups, each with a size of 1Gb. From 018c0359f3e2a90e9be9afce65d80930ea3ad863 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 8 Jun 2024 13:29:34 +0930 Subject: [PATCH 02/35] btrfs-progs: fix the conflicting super block flags [BUG] There is a bug report that a canceled checksum conversion (still experimental feature) resulted in unexpected super flags: csum_type 0 (crc32c) csum_size 4 csum 0x14973811 [match] bytenr 65536 flags 0x1000000001 ( WRITTEN | CHANGING_FSID_V2 ) magic _BHRfS_M [match] While for a filesystem under checksum conversion it should have either CHANGING_DATA_CSUM or CHANGING_META_CSUM. [CAUSE] It turns out that, due to btrfs-progs keeps its own extra flags inside its own ctree.h headers, not the shared uapi headers, we have conflicting super flags: kernel-shared/uapi/btrfs_tree.h:#define BTRFS_SUPER_FLAG_METADUMP_V2 (1ULL << 34) kernel-shared/uapi/btrfs_tree.h:#define BTRFS_SUPER_FLAG_CHANGING_FSID (1ULL << 35) kernel-shared/uapi/btrfs_tree.h:#define BTRFS_SUPER_FLAG_CHANGING_FSID_V2 (1ULL << 36) kernel-shared/ctree.h:#define BTRFS_SUPER_FLAG_CHANGING_DATA_CSUM (1ULL << 36) kernel-shared/ctree.h:#define BTRFS_SUPER_FLAG_CHANGING_META_CSUM (1ULL << 37) Note that CHANGING_FSID_V2 is conflicting with CHANGING_DATA_CSUM. [FIX] Cross port the proper updated uapi headers into btrfs-progs, and remove the definition from ctree.h. This would change the value for CHANGING_DATA_CSUM and CHANGING_META_CSUM, but considering they are experimental features, and kernel would reject them anyway, the damage is not that huge and we can accept such change before exposing it to end users. Pull-request: #810 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- kernel-shared/ctree.h | 10 ---------- kernel-shared/uapi/btrfs_tree.h | 7 +++++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/kernel-shared/ctree.h b/kernel-shared/ctree.h index 444d206e0a..1341a41872 100644 --- a/kernel-shared/ctree.h +++ b/kernel-shared/ctree.h @@ -59,16 +59,6 @@ static inline unsigned long btrfs_chunk_item_size(int num_stripes) sizeof(struct btrfs_stripe) * (num_stripes - 1); } -#define BTRFS_SUPER_FLAG_CHANGING_DATA_CSUM (1ULL << 36) -#define BTRFS_SUPER_FLAG_CHANGING_META_CSUM (1ULL << 37) - -/* - * The fs is undergoing block group tree feature change. - * If no BLOCK_GROUP_TREE compat ro flag, it's changing from regular - * bg item in extent tree to new bg tree. - */ -#define BTRFS_SUPER_FLAG_CHANGING_BG_TREE (1ULL << 38) - static inline u32 __BTRFS_LEAF_DATA_SIZE(u32 nodesize) { return nodesize - sizeof(struct btrfs_header); diff --git a/kernel-shared/uapi/btrfs_tree.h b/kernel-shared/uapi/btrfs_tree.h index e2ac228bcc..271346258d 100644 --- a/kernel-shared/uapi/btrfs_tree.h +++ b/kernel-shared/uapi/btrfs_tree.h @@ -758,6 +758,13 @@ struct btrfs_free_space_header { #define BTRFS_SUPER_FLAG_CHANGING_FSID (1ULL << 35) #define BTRFS_SUPER_FLAG_CHANGING_FSID_V2 (1ULL << 36) +/* + * Temporaray flags utilized by btrfs-progs to do offline conversion. They are + * rejected by kernel. Keep them all here to avoid conflicts. + */ +#define BTRFS_SUPER_FLAG_CHANGING_BG_TREE (1ULL << 38) +#define BTRFS_SUPER_FLAG_CHANGING_DATA_CSUM (1ULL << 39) +#define BTRFS_SUPER_FLAG_CHANGING_META_CSUM (1ULL << 40) /* * items in the extent btree are used to record the objectid of the From dcfdf6f8481bb0abf28a7d969c9a513a1b0de4c3 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 8 Jun 2024 13:37:17 +0930 Subject: [PATCH 03/35] btrfs-progs: print-tree: handle all supported flags Although we already have a pretty good array defined for all super/compat_ro/incompat flags, we still rely on a manually defined mask to do the printing. This can lead to easy de-sync between the definition and the flags. Change it to automatically iterate through the array to calculate the flags, and add the remaining super flags. Pull-request: #810 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- kernel-shared/print-tree.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/kernel-shared/print-tree.c b/kernel-shared/print-tree.c index a7018cb724..568a84cb35 100644 --- a/kernel-shared/print-tree.c +++ b/kernel-shared/print-tree.c @@ -1950,18 +1950,13 @@ static struct readable_flag_entry super_flags_array[] = { DEF_SUPER_FLAG_ENTRY(CHANGING_FSID_V2), DEF_SUPER_FLAG_ENTRY(SEEDING), DEF_SUPER_FLAG_ENTRY(METADUMP), - DEF_SUPER_FLAG_ENTRY(METADUMP_V2) + DEF_SUPER_FLAG_ENTRY(METADUMP_V2), + DEF_SUPER_FLAG_ENTRY(CHANGING_BG_TREE), + DEF_SUPER_FLAG_ENTRY(CHANGING_DATA_CSUM), + DEF_SUPER_FLAG_ENTRY(CHANGING_META_CSUM), }; static const int super_flags_num = ARRAY_SIZE(super_flags_array); -#define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ - BTRFS_HEADER_FLAG_RELOC |\ - BTRFS_SUPER_FLAG_CHANGING_FSID |\ - BTRFS_SUPER_FLAG_CHANGING_FSID_V2 |\ - BTRFS_SUPER_FLAG_SEEDING |\ - BTRFS_SUPER_FLAG_METADUMP |\ - BTRFS_SUPER_FLAG_METADUMP_V2) - static void __print_readable_flag(u64 flag, struct readable_flag_entry *array, int array_size, u64 supported_flags) { @@ -1995,26 +1990,33 @@ static void __print_readable_flag(u64 flag, struct readable_flag_entry *array, static void print_readable_compat_ro_flag(u64 flag) { - /* - * We know about the FREE_SPACE_TREE{,_VALID} bits, but we don't - * actually support them yet. - */ + u64 print_flags = 0; + + for (int i = 0; i < compat_ro_flags_num; i++) + print_flags |= compat_ro_flags_array[i].bit; return __print_readable_flag(flag, compat_ro_flags_array, compat_ro_flags_num, - BTRFS_FEATURE_COMPAT_RO_SUPP); + print_flags); } static void print_readable_incompat_flag(u64 flag) { + u64 print_flags = 0; + + for (int i = 0; i < incompat_flags_num; i++) + print_flags |= incompat_flags_array[i].bit; return __print_readable_flag(flag, incompat_flags_array, - incompat_flags_num, - BTRFS_FEATURE_INCOMPAT_SUPP); + incompat_flags_num, print_flags); } static void print_readable_super_flag(u64 flag) { + u64 print_flags = 0; + + for (int i = 0; i < super_flags_num; i++) + print_flags |= super_flags_array[i].bit; return __print_readable_flag(flag, super_flags_array, - super_flags_num, BTRFS_SUPER_FLAG_SUPP); + super_flags_num, print_flags); } static void print_sys_chunk_array(struct btrfs_super_block *sb) From 935fdd1cdbf1cbadafc78a9fe03db8c02df9f94d Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 02:58:25 +0200 Subject: [PATCH 04/35] btrfs-progs: ci: switch workflow runner image to ubuntu-24.04 There's an update to CI hosted runners, https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md - kernel 6.8 - e2fsprogs 1.47 - gcc 13.2 - clang 18.1.3 Switch the workflow files to use it as ubuntu-latest still points to the 22.04 version. The updated versions let us avoid workarounds due to old version if e2fsprogs. The musl 32bit build seems to fail so pin the version to the last one where it's known to work. Signed-off-by: David Sterba --- .github/workflows/artifacts-static-build.yml | 2 +- .github/workflows/ci-build-test-fast.yml | 27 +++++++++++--------- .github/workflows/ci-build-test.yml | 27 +++++++++++--------- .github/workflows/ci-sanitize.yml | 4 +-- .github/workflows/coverage.yml | 2 +- .github/workflows/devel.yml | 16 ++++++------ .github/workflows/pull-request.yml | 2 +- .github/workflows/test.yml | 2 +- 8 files changed, 44 insertions(+), 38 deletions(-) diff --git a/.github/workflows/artifacts-static-build.yml b/.github/workflows/artifacts-static-build.yml index 4621df30e6..3522ccc2f9 100644 --- a/.github/workflows/artifacts-static-build.yml +++ b/.github/workflows/artifacts-static-build.yml @@ -10,7 +10,7 @@ on: - master jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo apt-get install -y pkg-config gcc liblzo2-dev libzstd-dev libblkid-dev uuid-dev zlib1g-dev libext2fs-dev e2fsprogs libudev-dev python3-sphinx libaio-dev liburing-dev diff --git a/.github/workflows/ci-build-test-fast.yml b/.github/workflows/ci-build-test-fast.yml index eee247c432..93c7812845 100644 --- a/.github/workflows/ci-build-test-fast.yml +++ b/.github/workflows/ci-build-test-fast.yml @@ -14,31 +14,31 @@ on: jobs: check-centos7: name: CI Centos 7 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Centos7 run: sudo docker run kdave/ci-centos-7-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-libudev check-centos8: name: CI Centos 8 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Centos8 run: sudo docker run kdave/ci-centos-8-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned --disable-libudev check-leap153: name: CI Leap 15.3 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Leap 15.3 run: sudo docker run kdave/ci-opensuse-leap-15.3-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned check-leap154: name: CI Leap 15.4 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Leap 15.4 run: sudo docker run kdave/ci-opensuse-leap-15.4-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned check-musl: name: CI Musl - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Musl run: sudo docker run kdave/ci-musl-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-backtrace --disable-libudev @@ -46,7 +46,10 @@ jobs: run: sudo docker run kdave/ci-musl-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-backtrace --disable-libudev --enable-experimental check-musl-32bit: name: CI Musl (32bit) - runs-on: ubuntu-latest + # Pin the version, on 24.04 it fails: + # make: *** pselect jobs pipe: Function not implemented. Stop. + # make: *** Waiting for unfinished jobs.... + runs-on: ubuntu-22.04 steps: - name: CI Musl (32bit) run: sudo docker run kdave/ci-musl-i386 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-backtrace --disable-libudev @@ -54,7 +57,7 @@ jobs: run: sudo docker run kdave/ci-musl-i386 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-backtrace --disable-libudev --enable-experimental check-tumbleweed: name: CI Tumbleweed - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Tumbleweed run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation @@ -62,7 +65,7 @@ jobs: run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --enable-experimental check-tumbleweed-libgcrypt: name: CI Tumbleweed (libgcrypt) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Tumbleweed (libgcrypt) run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=libgcrypt @@ -70,7 +73,7 @@ jobs: run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=libgcrypt --enable-experimental check-tumbleweed-libsodium: name: CI Tumbleweed (libsodium) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Tumbleweed (libsodium) run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=libsodium @@ -78,7 +81,7 @@ jobs: run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=libsodium --enable-experimental check-tumbleweed-libkcapi: name: CI Tumbleweed (libkcapi) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Tumbleweed (libkcapi) run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=libkcapi @@ -86,7 +89,7 @@ jobs: run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=libkcapi --enable-experimental check-tumbleweed-botan: name: CI Tumbleweed (Botan) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: CI Tumbleweed (Botan) run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=botan @@ -94,7 +97,7 @@ jobs: run: sudo docker run kdave/ci-opensuse-tumbleweed-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --with-crypto=botan --enable-experimental # check-tumbleweed-openssl: # name: CI Tumbleweed (OpenSSL) -# runs-on: ubuntu-latest +# runs-on: ubuntu-24.04 # steps: # - uses: actions/checkout@v4 # - name: CI Tumbleweed (OpenSSL) diff --git a/.github/workflows/ci-build-test.yml b/.github/workflows/ci-build-test.yml index 2432dcb244..e0bb7aad79 100644 --- a/.github/workflows/ci-build-test.yml +++ b/.github/workflows/ci-build-test.yml @@ -12,84 +12,87 @@ on: jobs: check-centos7: name: CI Centos 7 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Centos7 run: ci/ci-build-centos7 check-centos8: name: CI Centos 8 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Centos8 run: ci/ci-build-centos8 check-leap153: name: CI Leap 15.3 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Leap 15.3 run: ci/ci-build-leap153 check-leap154: name: CI Leap 15.4 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Leap 15.4 run: ci/ci-build-leap154 check-musl: name: CI Musl - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Musl run: ci/ci-build-musl check-musl-32bit: name: CI Musl (32bit) - runs-on: ubuntu-latest + # Pin the version, on 24.04 it fails: + # make: *** pselect jobs pipe: Function not implemented. Stop. + # make: *** Waiting for unfinished jobs.... + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: CI Musl (32bit) run: ci/ci-build-musl-i386 check-tumbleweed: name: CI Tumbleweed - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Tumbleweed run: ci/ci-build-tumbleweed check-tumbleweed-libgcrypt: name: CI Tumbleweed (libgcrypt) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Tumbleweed (libgcrypt) run: ci/ci-build-tumbleweed HEAD --with-crypto=libgcrypt check-tumbleweed-libsodium: name: CI Tumbleweed (libsodium) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Tumbleweed (libsodium) run: ci/ci-build-tumbleweed HEAD --with-crypto=libsodium check-tumbleweed-libkcapi: name: CI Tumbleweed (libkcapi) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Tumbleweed (libkcapi) run: ci/ci-build-tumbleweed HEAD --with-crypto=libkcapi check-tumbleweed-botan: name: CI Tumbleweed (Botan) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: CI Tumbleweed (Botan) run: ci/ci-build-tumbleweed HEAD --with-crypto=botan # check-tumbleweed-openssl: # name: CI Tumbleweed (OpenSSL) -# runs-on: ubuntu-latest +# runs-on: ubuntu-24.04 # steps: # - uses: actions/checkout@v4 # - name: CI Tumbleweed (OpenSSL) diff --git a/.github/workflows/ci-sanitize.yml b/.github/workflows/ci-sanitize.yml index 1a88aacbab..41f8c187c8 100644 --- a/.github/workflows/ci-sanitize.yml +++ b/.github/workflows/ci-sanitize.yml @@ -13,7 +13,7 @@ on: - master jobs: build-ubsan: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: uname -a @@ -53,7 +53,7 @@ jobs: path: "tests/*-results.txt" if-no-files-found: ignore build-asan: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: uname -a diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3aea8cd5f5..d281ee5073 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -9,7 +9,7 @@ on: - master jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: uname -a diff --git a/.github/workflows/devel.yml b/.github/workflows/devel.yml index aca6ed9755..cefc544083 100644 --- a/.github/workflows/devel.yml +++ b/.github/workflows/devel.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: compiler: [ gcc, clang ] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs @@ -31,7 +31,7 @@ jobs: run: make V=1 EXTRA_CFLAGS='-march=x86-64' btrfs.box.static build-musl: name: Build test on musl - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Musl build run: sudo docker run kdave/ci-musl-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-backtrace --disable-libudev @@ -40,7 +40,7 @@ jobs: strategy: matrix: compiler: [ gcc, clang ] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs @@ -65,7 +65,7 @@ jobs: run: make hash-vectest && ./hash-vectest test-mkfs: name: Test mkfs.btrfs - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs @@ -85,7 +85,7 @@ jobs: if-no-files-found: ignore test-check: name: Test btrfs check - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs @@ -107,7 +107,7 @@ jobs: if-no-files-found: ignore test-misc: name: Test misc - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs @@ -127,7 +127,7 @@ jobs: if-no-files-found: ignore test-convert: name: Test btrfs-convert - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs @@ -147,7 +147,7 @@ jobs: if-no-files-found: ignore test-other: name: Test cli, fuzz - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: sudo modprobe btrfs diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 954e1ee5ff..e73b1802ed 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -13,7 +13,7 @@ on: branches: [ "devel", "master" ] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: uname -a diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 795c5d42cf..2083f1b204 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ on: - devel-ci jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - run: uname -a From bc28f9b63aec37a750791e2fafb4406a54e08f14 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 04:05:14 +0200 Subject: [PATCH 05/35] btrfs-progs: ci: move Leap image to 15.6 The support of Leap 15.4 has ended, not much point to test build there so move it to 15.6 (upper bound). The lower bound is still 15.3 to catch potential backward compatibility problems. The hub image exists (https://hub.docker.com/r/kdave/ci-opensuse-leap-15.6-x86_64). Signed-off-by: David Sterba --- .github/workflows/ci-build-test-fast.yml | 8 ++++---- .github/workflows/ci-build-test.yml | 8 ++++---- ci/{ci-build-leap154 => ci-build-leap156} | 4 ++-- .../Dockerfile | 2 +- .../docker-build | 0 .../docker-run | 0 .../run-tests | 0 .../test-build | 0 8 files changed, 11 insertions(+), 11 deletions(-) rename ci/{ci-build-leap154 => ci-build-leap156} (91%) rename ci/images/{ci-openSUSE-Leap-15.4-x86_64 => ci-openSUSE-Leap-15.6-x86_64}/Dockerfile (98%) rename ci/images/{ci-openSUSE-Leap-15.4-x86_64 => ci-openSUSE-Leap-15.6-x86_64}/docker-build (100%) rename ci/images/{ci-openSUSE-Leap-15.4-x86_64 => ci-openSUSE-Leap-15.6-x86_64}/docker-run (100%) rename ci/images/{ci-openSUSE-Leap-15.4-x86_64 => ci-openSUSE-Leap-15.6-x86_64}/run-tests (100%) rename ci/images/{ci-openSUSE-Leap-15.4-x86_64 => ci-openSUSE-Leap-15.6-x86_64}/test-build (100%) diff --git a/.github/workflows/ci-build-test-fast.yml b/.github/workflows/ci-build-test-fast.yml index 93c7812845..1be2455544 100644 --- a/.github/workflows/ci-build-test-fast.yml +++ b/.github/workflows/ci-build-test-fast.yml @@ -30,12 +30,12 @@ jobs: steps: - name: CI Leap 15.3 run: sudo docker run kdave/ci-opensuse-leap-15.3-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned - check-leap154: - name: CI Leap 15.4 + check-leap156: + name: CI Leap 15.6 runs-on: ubuntu-24.04 steps: - - name: CI Leap 15.4 - run: sudo docker run kdave/ci-opensuse-leap-15.4-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned + - name: CI Leap 15.6 + run: sudo docker run kdave/ci-opensuse-leap-15.6-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned check-musl: name: CI Musl runs-on: ubuntu-24.04 diff --git a/.github/workflows/ci-build-test.yml b/.github/workflows/ci-build-test.yml index e0bb7aad79..c15d64413b 100644 --- a/.github/workflows/ci-build-test.yml +++ b/.github/workflows/ci-build-test.yml @@ -31,13 +31,13 @@ jobs: - uses: actions/checkout@v4 - name: CI Leap 15.3 run: ci/ci-build-leap153 - check-leap154: - name: CI Leap 15.4 + check-leap156: + name: CI Leap 15.6 runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - name: CI Leap 15.4 - run: ci/ci-build-leap154 + - name: CI Leap 15.6 + run: ci/ci-build-leap156 check-musl: name: CI Musl runs-on: ubuntu-24.04 diff --git a/ci/ci-build-leap154 b/ci/ci-build-leap156 similarity index 91% rename from ci/ci-build-leap154 rename to ci/ci-build-leap156 index 3f54a5e37a..2dbdfc5977 100755 --- a/ci/ci-build-leap154 +++ b/ci/ci-build-leap156 @@ -1,6 +1,6 @@ #!/bin/sh # Usage: $0 [branch] [configure options] -# Create source tarball from HEAD or given branch and build it in openSUSE Leap 15.4 CI +# Create source tarball from HEAD or given branch and build it in openSUSE Leap 15.6 CI # environment. Configure options follow branch name that can be empty. HERE=`pwd` @@ -14,7 +14,7 @@ else exit 1 fi -CIIMAGEDIR=ci/images/ci-openSUSE-Leap-15.4-x86_64 +CIIMAGEDIR=ci/images/ci-openSUSE-Leap-15.6-x86_64 BRANCH=${1:-HEAD} if [ "$#" -ge 1 ]; then shift diff --git a/ci/images/ci-openSUSE-Leap-15.4-x86_64/Dockerfile b/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile similarity index 98% rename from ci/images/ci-openSUSE-Leap-15.4-x86_64/Dockerfile rename to ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile index f657cf189c..df8af2466d 100644 --- a/ci/images/ci-openSUSE-Leap-15.4-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile @@ -1,4 +1,4 @@ -FROM opensuse/leap:15.4 +FROM opensuse/leap:15.6 WORKDIR /tmp diff --git a/ci/images/ci-openSUSE-Leap-15.4-x86_64/docker-build b/ci/images/ci-openSUSE-Leap-15.6-x86_64/docker-build similarity index 100% rename from ci/images/ci-openSUSE-Leap-15.4-x86_64/docker-build rename to ci/images/ci-openSUSE-Leap-15.6-x86_64/docker-build diff --git a/ci/images/ci-openSUSE-Leap-15.4-x86_64/docker-run b/ci/images/ci-openSUSE-Leap-15.6-x86_64/docker-run similarity index 100% rename from ci/images/ci-openSUSE-Leap-15.4-x86_64/docker-run rename to ci/images/ci-openSUSE-Leap-15.6-x86_64/docker-run diff --git a/ci/images/ci-openSUSE-Leap-15.4-x86_64/run-tests b/ci/images/ci-openSUSE-Leap-15.6-x86_64/run-tests similarity index 100% rename from ci/images/ci-openSUSE-Leap-15.4-x86_64/run-tests rename to ci/images/ci-openSUSE-Leap-15.6-x86_64/run-tests diff --git a/ci/images/ci-openSUSE-Leap-15.4-x86_64/test-build b/ci/images/ci-openSUSE-Leap-15.6-x86_64/test-build similarity index 100% rename from ci/images/ci-openSUSE-Leap-15.4-x86_64/test-build rename to ci/images/ci-openSUSE-Leap-15.6-x86_64/test-build From d4d751753bbf930e9f3823eb1810bddbab45be02 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 01:39:06 +0200 Subject: [PATCH 06/35] btrfs-progs: tests: update or fix shell script coding style Fix the following issues in the test suite: - lack of quoting for variables - declare function variables local when missing (prevent accidental overwrite of global variables) - for variables with underscore in the name use plain "$VAR_NAME" instead of { } (unless necessary) - minor style adjustments like moving quotes to the end of the same string Signed-off-by: David Sterba --- tests/cli-tests/012-fi-du-recursion/test.sh | 2 +- tests/cli-tests/016-btrfs-fi-usage/test.sh | 14 +++++++------- tests/cli-tests/017-fi-show-missing/test.sh | 2 +- tests/common | 12 ++++++------ .../005-delete-all-rollback/test.sh | 2 +- .../025-ext4-uninit-written/test.sh | 2 +- tests/fsck-tests/012-leaf-corruption/test.sh | 19 ++++++++++++------- .../fsck-tests/024-clear-space-cache/test.sh | 2 +- .../037-freespacetree-repair/test.sh | 4 ++-- tests/fsck-tests/060-degraded-check/test.sh | 12 ++++++------ .../003-multi-check-unmounted/test.sh | 2 +- .../fuzz-tests/005-simple-dump-super/test.sh | 2 +- .../007-simple-super-recover/test.sh | 2 +- .../008-simple-chunk-recover/test.sh | 2 +- tests/fuzz-tests/009-simple-zero-log/test.sh | 2 +- tests/misc-tests/002-uuid-rewrite/test.sh | 2 ++ tests/misc-tests/003-zero-log/test.sh | 3 +++ .../009-subvolume-sync-must-wait/test.sh | 4 ++-- .../011-delete-missing-device/test.sh | 3 ++- .../013-subvolume-sync-crash/test.sh | 4 ++-- tests/misc-tests/016-send-clone-src/test.sh | 8 ++++---- .../misc-tests/018-recv-end-of-stream/test.sh | 9 +++++---- .../020-fix-superblock-corruption/test.sh | 2 ++ .../misc-tests/028-superblock-recover/test.sh | 6 +++--- .../030-missing-device-image/test.sh | 2 +- tests/misc-tests/034-metadata-uuid/test.sh | 11 ++++------- .../038-backup-root-corruption/test.sh | 2 +- tests/misc-tests/046-seed-multi-mount/test.sh | 7 +++++-- .../053-receive-write-encoded/test.sh | 3 ++- .../test.sh | 4 ++-- tests/misc-tests/060-ino-cache-clean/test.sh | 14 +++++++------- .../063-btrfstune-zoned-bgt/test.sh | 12 ++++++------ tests/mkfs-tests/001-basic-profiles/test.sh | 1 + .../test.sh | 6 ++++-- .../013-reserved-1M-for-single/test.sh | 2 ++ .../014-rootdir-inline-extent/test.sh | 2 +- tests/mkfs-tests/031-zoned-bgt/test.sh | 2 +- tests/mkfs-tests/032-zoned-reset/test.sh | 14 +++++++------- 38 files changed, 112 insertions(+), 92 deletions(-) diff --git a/tests/cli-tests/012-fi-du-recursion/test.sh b/tests/cli-tests/012-fi-du-recursion/test.sh index 5de3ab1d61..9b734043bf 100755 --- a/tests/cli-tests/012-fi-du-recursion/test.sh +++ b/tests/cli-tests/012-fi-du-recursion/test.sh @@ -20,7 +20,7 @@ _mktemp_local img2 1G run_check $SUDO_HELPER mkdir -p "$TEST_MNT"/mnt2 run_check $SUDO_HELPER dd if=/dev/zero bs=1M count=10 of="$TEST_MNT"/mnt2/hiddenfile -run_check $SUDO_HELPER "$TOP"/mkfs.btrfs -f img2 +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f img2 run_check $SUDO_HELPER mount -o loop img2 "$TEST_MNT"/mnt2 run_check $SUDO_HELPER dd if=/dev/zero bs=1M count=10 of="$TEST_MNT"/mnt2/file21 run_check $SUDO_HELPER dd if=/dev/zero bs=1M count=10 of="$TEST_MNT"/mnt2/file22 diff --git a/tests/cli-tests/016-btrfs-fi-usage/test.sh b/tests/cli-tests/016-btrfs-fi-usage/test.sh index f3a6f1acf6..ff1f97881a 100755 --- a/tests/cli-tests/016-btrfs-fi-usage/test.sh +++ b/tests/cli-tests/016-btrfs-fi-usage/test.sh @@ -63,14 +63,14 @@ test_raid1() set -- $i IFS=$OLDIFS - run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d$1 ${loopdevs[@]} + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d"$1" ${loopdevs[@]} run_check_mount_test_dev vars=($(report_numbers)) data_chunk_size=${vars[1]} used_on_dev=${vars[2]} data_ratio=${vars[0]} - [[ $used_on_dev -eq $data_chunk_size ]] || + [[ "$used_on_dev" -eq "$data_chunk_size" ]] || _fail "$1 inconsistent chunk/device usage. Chunk: $data_chunk_size Device: $used_on_dev" [[ "$data_ratio" = "$2" ]] || @@ -95,10 +95,10 @@ test_raid0() data_ratio=${vars[0]} # Divide by 4 since 4 loopp devices are setup - [[ $used_on_dev -eq $(($data_chunk_size / 4)) ]] || + [[ "$used_on_dev" -eq $(($data_chunk_size / 4)) ]] || _fail "raid0 inconsistent chunk/device usage. Chunk: $data_chunk_size Device: $used_on_dev" - [[ $data_ratio = "1.00" ]] || + [[ "$data_ratio" = "1.00" ]] || _fail "raid0: Unexpected data ratio: $data_ratio (must be 1.5)" run_check_umount_test_dev } @@ -118,17 +118,17 @@ test_raid56() set -- $i IFS=$OLDIFS - run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d$1 ${loopdevs[@]} + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d"$1" ${loopdevs[@]} run_check_mount_test_dev vars=($(report_numbers)) data_chunk_size=${vars[1]} used_on_dev=${vars[2]} data_ratio=${vars[0]} - [[ $used_on_dev -eq $(($data_chunk_size / $3)) ]] || + [[ "$used_on_dev" -eq $(($data_chunk_size / $3)) ]] || _fail "$i inconsistent chunk/device usage. Chunk: $data_chunk_size Device: $used_on_dev" - [[ $data_ratio = "$2" ]] || + [[ "$data_ratio" = "$2" ]] || _fail "$1: Unexpected data ratio: $data_ratio (must be $2)" run_check_umount_test_dev diff --git a/tests/cli-tests/017-fi-show-missing/test.sh b/tests/cli-tests/017-fi-show-missing/test.sh index 653bde3236..087415dd4c 100755 --- a/tests/cli-tests/017-fi-show-missing/test.sh +++ b/tests/cli-tests/017-fi-show-missing/test.sh @@ -21,7 +21,7 @@ run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d raid1 "${loopdevs[@]}" run_check $SUDO_HELPER mv "$dev2" /dev/loop-non-existent cond_wait_for_loopdevs -run_check $SUDO_HELPER mount -o degraded $dev1 $TEST_MNT +run_check $SUDO_HELPER mount -o degraded "$dev1" "$TEST_MNT" if ! run_check_stdout $SUDO_HELPER "$TOP/btrfs" filesystem show "$TEST_MNT" | \ grep -q -e "devid[[:space:]]\+2.*MISSING"; then diff --git a/tests/common b/tests/common index 7a23f6dde1..30a3201e50 100644 --- a/tests/common +++ b/tests/common @@ -739,9 +739,9 @@ _get_subvolid() DATASET_SIZE=50 generate_dataset() { + local dataset_type="$1" + local dirpath="$TEST_MNT/$dataset_type" - dataset_type="$1" - dirpath=$TEST_MNT/$dataset_type run_check $SUDO_HELPER mkdir -p "$dirpath" case "$dataset_type" in @@ -843,7 +843,7 @@ setup_loopdevs() # create all loop devices from a given loopdev environment prepare_loopdevs() { - for i in `seq $nloopdevs`; do + for i in `seq "$nloopdevs"`; do touch "$loopdev_prefix$i" chmod a+rw "$loopdev_prefix$i" truncate -s0 "$loopdev_prefix$i" @@ -921,14 +921,14 @@ prepare_nullbdevs() # Record any other pre-existing devices in case creation fails run_check $SUDO_HELPER "$nullb" ls - for i in `seq ${nullb_count}`; do + for i in `seq "$nullb_count"`; do # Last line has the name of the device node path - out=$(run_check_stdout $SUDO_HELPER "$nullb" create -s "${nullb_size}" -z "${nullb_zone_size}") + out=$(run_check_stdout $SUDO_HELPER "$nullb" create -s "$nullb_size" -z "$nullb_zone_size") if [ $? != 0 ]; then _fail "cannot create nullb zoned device $i" fi dev=$(echo "$out" | tail -n 1) - nullb_devs[$i]=${dev} + nullb_devs[$i]="$dev" done run_check $SUDO_HELPER "$nullb" ls diff --git a/tests/convert-tests/005-delete-all-rollback/test.sh b/tests/convert-tests/005-delete-all-rollback/test.sh index 5603d3078b..d0be331ad4 100755 --- a/tests/convert-tests/005-delete-all-rollback/test.sh +++ b/tests/convert-tests/005-delete-all-rollback/test.sh @@ -63,7 +63,7 @@ do_test() { convert_test_post_check_checksums "$CHECKSUMTMP" run_check_umount_test_dev - rm "$CHECKSUMTMP" + rm -- "$CHECKSUMTMP" } # Iterate over defaults and options that are not tied to hardware capabilities diff --git a/tests/convert-tests/025-ext4-uninit-written/test.sh b/tests/convert-tests/025-ext4-uninit-written/test.sh index cca8f69a20..7230f232d7 100755 --- a/tests/convert-tests/025-ext4-uninit-written/test.sh +++ b/tests/convert-tests/025-ext4-uninit-written/test.sh @@ -29,7 +29,7 @@ fi # Now fill the underlying range with non-zeros. # For properly converted fs, we should not read the contents anyway -run_check $SUDO_HELPER dd if=/dev/urandom of=$TEST_DEV bs=4096 seek="$physical" conv=notrunc count=8 +run_check $SUDO_HELPER dd if=/dev/urandom of="$TEST_DEV" bs=4096 seek="$physical" conv=notrunc count=8 # Write some thing into the file range. run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file" bs=4096 count=1 conv=notrunc diff --git a/tests/fsck-tests/012-leaf-corruption/test.sh b/tests/fsck-tests/012-leaf-corruption/test.sh index 5539d8ab68..dd849f853d 100755 --- a/tests/fsck-tests/012-leaf-corruption/test.sh +++ b/tests/fsck-tests/012-leaf-corruption/test.sh @@ -48,11 +48,14 @@ generate_leaf_corrupt_no_data_ext() check_inode() { - path=$1 - ino=$2 - size=$3 - mode=$4 - name=$5 + local path=$1 + local ino=$2 + local size=$3 + local mode=$4 + local name=$5 + local exists + local found_mode + local found_size # Check whether the inode exists exists=$($SUDO_HELPER find "$path" -inum "$ino") @@ -84,11 +87,13 @@ check_inode() # Check salvaged data in the recovered image check_leaf_corrupt_no_data_ext() { - image=$1 + local image=$1 + local i + $SUDO_HELPER mount -o loop -t btrfs "$image" -o ro "$TEST_MNT" i=0 - while [ $i -lt ${#leaf_no_data_ext_list[@]} ]; do + while [ "$i" -lt ${#leaf_no_data_ext_list[@]} ]; do check_inode "$TEST_MNT/lost+found" \ ${leaf_no_data_ext_list[i]} \ ${leaf_no_data_ext_list[i + 1]} \ diff --git a/tests/fsck-tests/024-clear-space-cache/test.sh b/tests/fsck-tests/024-clear-space-cache/test.sh index 848ec46c1b..800b2619c5 100755 --- a/tests/fsck-tests/024-clear-space-cache/test.sh +++ b/tests/fsck-tests/024-clear-space-cache/test.sh @@ -23,7 +23,7 @@ sync # Remove file 1 3 5 to create holes for i in 1 3 5; do - run_check $SUDO_HELPER rm "$TEST_MNT/file_${i}" + run_check $SUDO_HELPER rm "$TEST_MNT/file_$i" done sync diff --git a/tests/fsck-tests/037-freespacetree-repair/test.sh b/tests/fsck-tests/037-freespacetree-repair/test.sh index 9e3d7dbcd0..e615fd5e6e 100755 --- a/tests/fsck-tests/037-freespacetree-repair/test.sh +++ b/tests/fsck-tests/037-freespacetree-repair/test.sh @@ -27,7 +27,7 @@ corrupt_fst_item() local offset type="$1" - if [[ $type == "bitmap" ]]; then + if [[ "$type" == "bitmap" ]]; then type=200 objectid=$("$TOP/btrfs" inspect-internal dump-tree -t 10 "$TEST_DEV" | \ grep -o "[[:digit:]]* FREE_SPACE_BITMAP [[:digit:]]*" | \ @@ -40,7 +40,7 @@ corrupt_fst_item() return 1 fi _log "Corrupting $objectid,FREE_SPACE_BITMAP,$offset" - elif [[ $type == "extent" ]]; then + elif [[ "$type" == "extent" ]]; then type=199 objectid=$("$TOP/btrfs" inspect-internal dump-tree -t 10 "$TEST_DEV" | \ grep -o "[[:digit:]]* FREE_SPACE_EXTENT [[:digit:]]*" | \ diff --git a/tests/fsck-tests/060-degraded-check/test.sh b/tests/fsck-tests/060-degraded-check/test.sh index 1fb0cacdae..c4a91cc5cf 100755 --- a/tests/fsck-tests/060-degraded-check/test.sh +++ b/tests/fsck-tests/060-degraded-check/test.sh @@ -20,17 +20,17 @@ dev3=${loopdevs[3]} # Run 1: victim is dev1 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -m raid5 -d raid5 "${loopdevs[@]}" -run_check $SUDO_HELPER wipefs -fa $dev1 -run_check $SUDO_HELPER "$TOP/btrfs" check $dev2 +run_check $SUDO_HELPER wipefs -fa "$dev1" +run_check $SUDO_HELPER "$TOP/btrfs" check "$dev2" # Run 2: victim is dev2 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -m raid5 -d raid5 "${loopdevs[@]}" -run_check $SUDO_HELPER wipefs -fa $dev2 -run_check $SUDO_HELPER "$TOP/btrfs" check $dev3 +run_check $SUDO_HELPER wipefs -fa "$dev2" +run_check $SUDO_HELPER "$TOP/btrfs" check "$dev3" # Run 3: victim is dev3 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -m raid5 -d raid5 "${loopdevs[@]}" -run_check $SUDO_HELPER wipefs -fa $dev3 -run_check $SUDO_HELPER "$TOP/btrfs" check $dev1 +run_check $SUDO_HELPER wipefs -fa "$dev3" +run_check $SUDO_HELPER "$TOP/btrfs" check "$dev1" cleanup_loopdevs diff --git a/tests/fuzz-tests/003-multi-check-unmounted/test.sh b/tests/fuzz-tests/003-multi-check-unmounted/test.sh index 472b80d6fc..9b49ca073a 100755 --- a/tests/fuzz-tests/003-multi-check-unmounted/test.sh +++ b/tests/fuzz-tests/003-multi-check-unmounted/test.sh @@ -13,7 +13,7 @@ setup_root_helper check_image() { local image - image=$1 + image="$1" run_mayfail $TOP/btrfs check -s 1 "$image" run_mayfail $TOP/btrfs check --force --init-csum-tree "$image" run_mayfail $TOP/btrfs check --repair --force --init-extent-tree "$image" diff --git a/tests/fuzz-tests/005-simple-dump-super/test.sh b/tests/fuzz-tests/005-simple-dump-super/test.sh index 59ae702f0b..fe9acd43ac 100755 --- a/tests/fuzz-tests/005-simple-dump-super/test.sh +++ b/tests/fuzz-tests/005-simple-dump-super/test.sh @@ -10,7 +10,7 @@ setup_root_helper check_image() { local image - image=$1 + image="$1" run_mayfail "$TOP/btrfs" inspect-internal dump-super "$image" run_mayfail "$TOP/btrfs" inspect-internal dump-super -Ffa "$image" } diff --git a/tests/fuzz-tests/007-simple-super-recover/test.sh b/tests/fuzz-tests/007-simple-super-recover/test.sh index ceee5a4ce3..3ba375536c 100755 --- a/tests/fuzz-tests/007-simple-super-recover/test.sh +++ b/tests/fuzz-tests/007-simple-super-recover/test.sh @@ -16,6 +16,6 @@ check_image() { rm -- "$image".scratch } -check_all_images $TEST_TOP/fuzz-tests/images +check_all_images "$TEST_TOP/fuzz-tests/images" exit 0 diff --git a/tests/fuzz-tests/008-simple-chunk-recover/test.sh b/tests/fuzz-tests/008-simple-chunk-recover/test.sh index 9ef2c364b7..c972481848 100755 --- a/tests/fuzz-tests/008-simple-chunk-recover/test.sh +++ b/tests/fuzz-tests/008-simple-chunk-recover/test.sh @@ -10,7 +10,7 @@ setup_root_helper check_image() { local image - image=$1 + image="$1" run_check cp "$image" "$image".scratch run_mayfail "$TOP/btrfs" rescue chunk-recover -y -v "$image".scratch rm -- "$image".scratch diff --git a/tests/fuzz-tests/009-simple-zero-log/test.sh b/tests/fuzz-tests/009-simple-zero-log/test.sh index d62f4d4571..013c435af8 100755 --- a/tests/fuzz-tests/009-simple-zero-log/test.sh +++ b/tests/fuzz-tests/009-simple-zero-log/test.sh @@ -10,7 +10,7 @@ setup_root_helper check_image() { local image - image=$1 + image="$1" run_check cp "$image" "$image".scratch run_mayfail "$TOP/btrfs" rescue zero-log "$image".scratch rm -- "$image".scratch diff --git a/tests/misc-tests/002-uuid-rewrite/test.sh b/tests/misc-tests/002-uuid-rewrite/test.sh index 4c24362b63..9ecac63110 100755 --- a/tests/misc-tests/002-uuid-rewrite/test.sh +++ b/tests/misc-tests/002-uuid-rewrite/test.sh @@ -17,6 +17,7 @@ get_fs_uuid() { test_uuid_random() { local origuuid + local currentfsid origuuid=11111111-a101-4031-b29a-379d4f8b7a2d @@ -37,6 +38,7 @@ test_uuid_user() { local origuuid local newuuid + local fsid origuuid=22222222-d324-4f92-80e9-7658bf3b845f newuuid=33333333-bfc9-4045-9399-a396dc6893b3 diff --git a/tests/misc-tests/003-zero-log/test.sh b/tests/misc-tests/003-zero-log/test.sh index 5c11ff7594..b647e4dd5e 100755 --- a/tests/misc-tests/003-zero-log/test.sh +++ b/tests/misc-tests/003-zero-log/test.sh @@ -19,6 +19,9 @@ get_log_root_level() { test_zero_log() { + local log_root + local log_root_level + # FIXME: we need an image with existing log_root run_check_mkfs_test_dev --rootdir "$INTERNAL_BIN/Documentation" run_check "$TOP/btrfs" inspect-internal dump-super "$TEST_DEV" diff --git a/tests/misc-tests/009-subvolume-sync-must-wait/test.sh b/tests/misc-tests/009-subvolume-sync-must-wait/test.sh index 22de6940ea..126a5fae0a 100755 --- a/tests/misc-tests/009-subvolume-sync-must-wait/test.sh +++ b/tests/misc-tests/009-subvolume-sync-must-wait/test.sh @@ -18,11 +18,11 @@ run_check $SUDO_HELPER chmod a+rw "$TEST_MNT" cd "$TEST_MNT" for i in `seq 5`; do - run_check dd if=/dev/zero of=file$i bs=1M count=10 + run_check dd if=/dev/zero of="file$i" bs=1M count=10 done for sn in `seq 4`;do - run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot . snap$sn + run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot . "snap$sn" for i in `seq 10`; do run_check dd if=/dev/zero of="snap$sn/file$i" bs=1M count=10 done diff --git a/tests/misc-tests/011-delete-missing-device/test.sh b/tests/misc-tests/011-delete-missing-device/test.sh index 5f41b2ed7d..0085fa1979 100755 --- a/tests/misc-tests/011-delete-missing-device/test.sh +++ b/tests/misc-tests/011-delete-missing-device/test.sh @@ -26,6 +26,8 @@ test_wipefs() } test_delete_missing() { + local out + run_check_mount_test_dev -o degraded run_check $SUDO_HELPER "$TOP/btrfs" filesystem show "$TEST_MNT" run_check $SUDO_HELPER "$TOP/btrfs" device delete missing "$TEST_MNT" @@ -33,7 +35,6 @@ test_delete_missing() run_check_umount_test_dev run_check_mount_test_dev - local out out=$(run_check_stdout $SUDO_HELPER "$TOP/btrfs" filesystem show "$TEST_MNT") if echo "$out" | grep -q -- "$devtodel"; then _fail "device $devtodel not deleted" diff --git a/tests/misc-tests/013-subvolume-sync-crash/test.sh b/tests/misc-tests/013-subvolume-sync-crash/test.sh index ec34081e57..e469b9aea9 100755 --- a/tests/misc-tests/013-subvolume-sync-crash/test.sh +++ b/tests/misc-tests/013-subvolume-sync-crash/test.sh @@ -18,12 +18,12 @@ run_check $SUDO_HELPER chmod a+rw "$TEST_MNT" cd "$TEST_MNT" for i in `seq 5`; do - run_check dd if=/dev/zero of=file$i bs=1M count=10 + run_check dd if=/dev/zero of="file$i" bs=1M count=10 done # 128 is minimum for sn in `seq 130`;do - run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot . snap$sn + run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot . "snap$sn" for i in `seq 10`; do run_check dd if=/dev/zero of="snap$sn/file$i" bs=1M count=1 done diff --git a/tests/misc-tests/016-send-clone-src/test.sh b/tests/misc-tests/016-send-clone-src/test.sh index 686bb36ad2..84c492cc86 100755 --- a/tests/misc-tests/016-send-clone-src/test.sh +++ b/tests/misc-tests/016-send-clone-src/test.sh @@ -19,14 +19,14 @@ cd "$TEST_MNT" || _fail "cannot chdir to TEST_MNT" run_check $SUDO_HELPER "$TOP/btrfs" subvolume create subv-parent1 for i in 1 2 3; do - run_check $SUDO_HELPER dd if=/dev/zero of=subv-parent1/file1_$i bs=1M count=1 - run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv-parent1 subv-snap1_$i + run_check $SUDO_HELPER dd if=/dev/zero of="subv-parent1/file1_$i" bs=1M count=1 + run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv-parent1 "subv-snap1_$i" done run_check $SUDO_HELPER "$TOP/btrfs" subvolume create subv-parent2 for i in 1 2 3; do - run_check $SUDO_HELPER dd if=/dev/zero of=subv-parent2/file2_$i bs=1M count=1 - run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv-parent2 subv-snap2_$i + run_check $SUDO_HELPER dd if=/dev/zero of="subv-parent2/file2_$i" bs=1M count=1 + run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv-parent2 "subv-snap2_$i" done _mktemp_local "$here/send-stream.img" diff --git a/tests/misc-tests/018-recv-end-of-stream/test.sh b/tests/misc-tests/018-recv-end-of-stream/test.sh index 5e5d1540a7..ece87ec92d 100755 --- a/tests/misc-tests/018-recv-end-of-stream/test.sh +++ b/tests/misc-tests/018-recv-end-of-stream/test.sh @@ -52,7 +52,7 @@ test_full_simple_stream() { run_check $SUDO_HELPER "$TOP/btrfs" subvolume create subv1 for i in 1 2 3; do - run_check $SUDO_HELPER dd if=/dev/zero of=subv1/file1_$i bs=1M count=1 + run_check $SUDO_HELPER dd if=/dev/zero of="subv1/file1_$i" bs=1M count=1 done run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv1 subv1-snap @@ -104,7 +104,8 @@ test_incr_empty_stream() { } test_incr_simple_stream() { - local str + local fstr + local istr fstr="$here/stream-full-simple.stream" istr="$here/stream-incr-simple.stream" @@ -115,13 +116,13 @@ test_incr_simple_stream() { run_check $SUDO_HELPER "$TOP/btrfs" subvolume create subv1 for i in 1 2 3; do - run_check $SUDO_HELPER dd if=/dev/zero of=subv1/file1_$i bs=1M count=1 + run_check $SUDO_HELPER dd if=/dev/zero of="subv1/file1_$i" bs=1M count=1 done run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv1 subv1-snap for i in 1 2 3; do - run_check $SUDO_HELPER dd if=/dev/urandom of=subv1/file1_$i bs=1M count=1 + run_check $SUDO_HELPER dd if=/dev/urandom of="subv1/file1_$i" bs=1M count=1 done run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r subv1 subv2-snap diff --git a/tests/misc-tests/020-fix-superblock-corruption/test.sh b/tests/misc-tests/020-fix-superblock-corruption/test.sh index e904160af8..404fbb78d0 100755 --- a/tests/misc-tests/020-fix-superblock-corruption/test.sh +++ b/tests/misc-tests/020-fix-superblock-corruption/test.sh @@ -14,6 +14,8 @@ FIRST_SUPERBLOCK_OFFSET=65536 test_superblock_restore() { + local seek + run_check_mkfs_test_dev # Corrupt superblock checksum diff --git a/tests/misc-tests/028-superblock-recover/test.sh b/tests/misc-tests/028-superblock-recover/test.sh index 646ec6eac6..474ca03baa 100755 --- a/tests/misc-tests/028-superblock-recover/test.sh +++ b/tests/misc-tests/028-superblock-recover/test.sh @@ -13,8 +13,8 @@ prepare_test_dev 260G run_check_mkfs_test_dev check_corruption() { - local sb_offset=$1 - local source_sb=$2 + local sb_offset="$1" + local source_sb="$2" # First we ensure we can mount it successfully run_check_mount_test_dev @@ -37,7 +37,7 @@ check_corruption() { # Now run btrfs rescue which should fix the superblock. It uses 2 # to signal success of recovery use mayfail to ignore that retval # but still log the output of the command - run_mayfail $SUDO_HELPER "$TOP"/btrfs rescue super-recover -yv "$TEST_DEV" + run_mayfail $SUDO_HELPER "$TOP/btrfs" rescue super-recover -yv "$TEST_DEV" if [ $? != 2 ]; then _fail "couldn't rescue super" fi diff --git a/tests/misc-tests/030-missing-device-image/test.sh b/tests/misc-tests/030-missing-device-image/test.sh index 588d3e8831..c7f1f629c8 100755 --- a/tests/misc-tests/030-missing-device-image/test.sh +++ b/tests/misc-tests/030-missing-device-image/test.sh @@ -24,7 +24,7 @@ test_missing() local good_num local good_dev - bad_num=$1 + bad_num="$1" bad_dev=${loopdevs[$bad_num]} good_num=$((3 - $bad_num)) good_dev=${loopdevs[$good_num]} diff --git a/tests/misc-tests/034-metadata-uuid/test.sh b/tests/misc-tests/034-metadata-uuid/test.sh index 43bab324fa..9dcb894890 100755 --- a/tests/misc-tests/034-metadata-uuid/test.sh +++ b/tests/misc-tests/034-metadata-uuid/test.sh @@ -43,13 +43,11 @@ read_metadata_uuid() { echo $(run_check_stdout $SUDO_HELPER "$TOP/btrfs" inspect-internal \ dump-super "$dev" | awk '/metadata_uuid/ {print $2}') else - read_fsid $dev + read_fsid "$dev" fi } check_btrfstune() { - local fsid - _log "Checking btrfstune logic" # test with random uuid run_check $SUDO_HELPER "$TOP/btrfstune" -m "$TEST_DEV" @@ -89,17 +87,16 @@ check_dump_super_output() { # assert that metadata/fsid match on non-changed fs fsid=$(read_fsid "$TEST_DEV") metadata_uuid=$(read_metadata_uuid "$TEST_DEV") - [ "$fsid" = "$metadata_uuid" ] || _fail "fsid ("$fsid") doesn't match metadata_uuid ("$metadata_uuid")" + [ "$fsid" = "$metadata_uuid" ] || _fail "fsid ($fsid) doesn't match metadata_uuid ($metadata_uuid)" dev_item_match=$(run_check_stdout $SUDO_HELPER "$TOP/btrfs" inspect-internal dump-super \ "$TEST_DEV" | awk '/dev_item.fsid/ {print $3}') - [ $dev_item_match = "[match]" ] || _fail "dev_item.fsid doesn't match on non-metadata uuid fs" - + [ "$dev_item_match" = "[match]" ] || _fail "dev_item.fsid doesn't match on non-metadata uuid fs" _log "Checking output after fsid change" # change metadatauuid and ensure everything in the output is still correct - old_metadata_uuid=$metadata_uuid + old_metadata_uuid="$metadata_uuid" run_check $SUDO_HELPER "$TOP/btrfstune" -M d88c8333-a652-4476-b225-2e9284eb59f1 "$TEST_DEV" fsid=$(read_fsid "$TEST_DEV") metadata_uuid=$(read_metadata_uuid "$TEST_DEV") diff --git a/tests/misc-tests/038-backup-root-corruption/test.sh b/tests/misc-tests/038-backup-root-corruption/test.sh index d276386f88..d4396a7fe2 100755 --- a/tests/misc-tests/038-backup-root-corruption/test.sh +++ b/tests/misc-tests/038-backup-root-corruption/test.sh @@ -68,5 +68,5 @@ _log "After the backup usage:" _log "$(dump_super)" if [ "$main_root_ptr" -ne "$backup_new_root_ptr" ]; then - _fail "Backup ${slot_num} not overwritten" + _fail "Backup $slot_num not overwritten" fi diff --git a/tests/misc-tests/046-seed-multi-mount/test.sh b/tests/misc-tests/046-seed-multi-mount/test.sh index 38be425966..1ac80704f1 100755 --- a/tests/misc-tests/046-seed-multi-mount/test.sh +++ b/tests/misc-tests/046-seed-multi-mount/test.sh @@ -22,9 +22,9 @@ run_check_mkfs_test_dev -L BTRFS-TESTS-SEED run_check_mount_test_dev for i in `seq 6`; do - run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file${i}" bs=1M count=1 status=none + run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file$i" bs=1M count=1 status=none # Something to distinguish the contents - run_check md5sum "$TEST_MNT/file${i}" + run_check md5sum "$TEST_MNT/file$i" done run_check_umount_test_dev @@ -34,6 +34,9 @@ TEST_DEV=${loopdevs[1]} nextdevice() { local nextdev + local mnt + local md5sum + local md5sum2 nextdev="$1" # Mount again, as seeding device diff --git a/tests/misc-tests/053-receive-write-encoded/test.sh b/tests/misc-tests/053-receive-write-encoded/test.sh index bc06ee0a8e..95d9e97596 100755 --- a/tests/misc-tests/053-receive-write-encoded/test.sh +++ b/tests/misc-tests/053-receive-write-encoded/test.sh @@ -22,7 +22,8 @@ here=`pwd` # assumes the filesystem exists, and does mount, write, snapshot, send, unmount # for the specified encoding option send_one() { - local str + local algorithm + local file local subv local snap diff --git a/tests/misc-tests/059-qgroup-show-stale-qgroup-crash/test.sh b/tests/misc-tests/059-qgroup-show-stale-qgroup-crash/test.sh index bcc924b46d..cd7ffd2794 100755 --- a/tests/misc-tests/059-qgroup-show-stale-qgroup-crash/test.sh +++ b/tests/misc-tests/059-qgroup-show-stale-qgroup-crash/test.sh @@ -13,9 +13,9 @@ count=24 run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file bs=1M count=1 run_check $SUDO_HELPER "$TOP/btrfs" quota enable "$TEST_MNT" for i in `seq $count`; do - run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/subv${i}" + run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/subv$i" if [ "$(($i % 2))" = "0" ]; then - run_check $SUDO_HELPER "$TOP/btrfs" subvolume delete "$TEST_MNT/subv${i}" + run_check $SUDO_HELPER "$TOP/btrfs" subvolume delete "$TEST_MNT/subv$i" fi done run_check $SUDO_HELPER "$TOP/btrfs" qgroup show --sort path "$TEST_MNT" diff --git a/tests/misc-tests/060-ino-cache-clean/test.sh b/tests/misc-tests/060-ino-cache-clean/test.sh index 136f0bd32b..518e203582 100755 --- a/tests/misc-tests/060-ino-cache-clean/test.sh +++ b/tests/misc-tests/060-ino-cache-clean/test.sh @@ -15,29 +15,29 @@ run_check "$TOP/btrfs" check "$image" # Check for FREE_INO items for toplevel subvol item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t fs "$image" | grep -c 'item [0-9].* key (FREE_INO') -[ $item_count -eq 0 ] || _fail "FREE_INO items for toplevel subvolume present" +[ "$item_count" -eq 0 ] || _fail "FREE_INO items for toplevel subvolume present" # Check for bitmap item for toplevel subvol item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t fs "$image" | grep -c '(FREE_SPACE') -[ $item_count -eq 0 ] || _fail "FREE_SPACE items for toplevel subvolume present" +[ "$item_count" -eq 0 ] || _fail "FREE_SPACE items for toplevel subvolume present" # Check for FREE_INO items for subvolume item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t 256 "$image" | grep -c 'item [0-9].* key (FREE_INO') -[ $item_count -eq 0 ] || _fail "ino cache items for subvolume present" +[ "$item_count" -eq 0 ] || _fail "ino cache items for subvolume present" # Check for bitmap item for subvolume item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t 256 "$image" | grep -c '(FREE_SPACE') -[ $item_count -eq 0 ] || _fail "FREE_SPACE items for subvolume present" +[ "$item_count" -eq 0 ] || _fail "FREE_SPACE items for subvolume present" # Check for FREE_INO items for snapshot item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t 257 "$image" | grep -c 'item [0-9].* key (FREE_INO') -[ $item_count -eq 0 ] || _fail "ino cache items for snapshot present" +[ "$item_count" -eq 0 ] || _fail "ino cache items for snapshot present" # Check for bitmap item for snapshot item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t 257 "$image" | grep -c '(FREE_SPACE') -[ $item_count -eq 0 ] || _fail "FREE_SPACE items for snapshot present" +[ "$item_count" -eq 0 ] || _fail "FREE_SPACE items for snapshot present" # Finally test that the csum tree is empty as ino cache also uses it. At this # point all ino items/extents should have been deleted hence the csum tree should @@ -45,6 +45,6 @@ item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t 257 "$i item_count=$(run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t csum "$image" | sed -n -e 's/^.* items \([0-9]*\).*/\1/p') -[ $item_count -eq 0 ] || _fail "csum tree not empty" +[ "$item_count" -eq 0 ] || _fail "csum tree not empty" rm -f -- "$image" diff --git a/tests/misc-tests/063-btrfstune-zoned-bgt/test.sh b/tests/misc-tests/063-btrfstune-zoned-bgt/test.sh index dc2003cc78..5315f7476d 100755 --- a/tests/misc-tests/063-btrfstune-zoned-bgt/test.sh +++ b/tests/misc-tests/063-btrfstune-zoned-bgt/test.sh @@ -25,31 +25,31 @@ if [ $? != 0 ]; then _fail "cannot create nullb zoned device $i" fi dev=$(echo "$out" | tail -n 1) -name=$(basename "${dev}") +name=$(basename "$dev") run_check $SUDO_HELPER "$nullb" ls -TEST_DEV="${dev}" +TEST_DEV="$dev" # Create the fs without bgt -run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -m single -d single -O ^block-group-tree "${dev}" +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -m single -d single -O ^block-group-tree "$dev" run_check_mount_test_dev run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file1 bs=1M count=1 run_check $SUDO_HELPER "$TOP/btrfs" filesystem usage -T "$TEST_MNT" run_check_umount_test_dev # Convert to bgt -run_check $SUDO_HELPER "$TOP/btrfstune" --convert-to-block-group-tree "${dev}" +run_check $SUDO_HELPER "$TOP/btrfstune" --convert-to-block-group-tree "$dev" run_check_mount_test_dev run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file2 bs=1M count=1 run_check $SUDO_HELPER "$TOP/btrfs" filesystem usage -T "$TEST_MNT" run_check_umount_test_dev # And convert back to old extent tree -run_check $SUDO_HELPER "$TOP/btrfstune" --convert-from-block-group-tree "${dev}" +run_check $SUDO_HELPER "$TOP/btrfstune" --convert-from-block-group-tree "$dev" run_check_mount_test_dev run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file3 bs=1M count=1 run_check $SUDO_HELPER "$TOP/btrfs" filesystem usage -T "$TEST_MNT" run_check_umount_test_dev -run_check $SUDO_HELPER "$nullb" rm "${name}" +run_check $SUDO_HELPER "$nullb" rm "$name" diff --git a/tests/mkfs-tests/001-basic-profiles/test.sh b/tests/mkfs-tests/001-basic-profiles/test.sh index 07df6faa06..2b46f1ffa0 100755 --- a/tests/mkfs-tests/001-basic-profiles/test.sh +++ b/tests/mkfs-tests/001-basic-profiles/test.sh @@ -36,6 +36,7 @@ test_get_info() run_check $SUDO_HELPER "$TOP/btrfs" device usage "$TEST_MNT" run_check $SUDO_HELPER umount "$TEST_MNT" } + test_do_mkfs() { run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$@" diff --git a/tests/mkfs-tests/008-sectorsize-nodesize-combination/test.sh b/tests/mkfs-tests/008-sectorsize-nodesize-combination/test.sh index a7d8216fe4..fe137f3c95 100755 --- a/tests/mkfs-tests/008-sectorsize-nodesize-combination/test.sh +++ b/tests/mkfs-tests/008-sectorsize-nodesize-combination/test.sh @@ -17,8 +17,10 @@ features="^mixed-bg" # caller need to check whether the combination is valid do_test() { - sectorsize=$1 - nodesize=$2 + local sectorsize=$1 + local nodesize=$2 + local ret + run_mayfail "$TOP/mkfs.btrfs" -f -O "$features" -n "$nodesize" -s "$sectorsize" \ "$TEST_DEV" ret=$? diff --git a/tests/mkfs-tests/013-reserved-1M-for-single/test.sh b/tests/mkfs-tests/013-reserved-1M-for-single/test.sh index bc83a25503..9434f741ef 100755 --- a/tests/mkfs-tests/013-reserved-1M-for-single/test.sh +++ b/tests/mkfs-tests/013-reserved-1M-for-single/test.sh @@ -15,6 +15,8 @@ prepare_test_dev do_one_test () { + local first_dev_extent + run_check_mkfs_test_dev "$@" # Use dev-extent tree to find first device extent diff --git a/tests/mkfs-tests/014-rootdir-inline-extent/test.sh b/tests/mkfs-tests/014-rootdir-inline-extent/test.sh index 889e6eee61..7c686ae4d6 100755 --- a/tests/mkfs-tests/014-rootdir-inline-extent/test.sh +++ b/tests/mkfs-tests/014-rootdir-inline-extent/test.sh @@ -21,7 +21,7 @@ create_file() test_mkfs_rootdir() { - nodesize=$1 + local nodesize=$1 run_check_mkfs_test_dev --nodesize "$nodesize" --rootdir "$tmp" run_check $SUDO_HELPER "$TOP/btrfs" check "$TEST_DEV" } diff --git a/tests/mkfs-tests/031-zoned-bgt/test.sh b/tests/mkfs-tests/031-zoned-bgt/test.sh index e296c29b92..97aa885fc3 100755 --- a/tests/mkfs-tests/031-zoned-bgt/test.sh +++ b/tests/mkfs-tests/031-zoned-bgt/test.sh @@ -11,7 +11,7 @@ prepare_nullbdevs TEST_DEV="${nullb_devs[1]}" # Use single as it's supported on more kernels -run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -m single -d single -O block-group-tree "${TEST_DEV}" +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -m single -d single -O block-group-tree "$TEST_DEV" run_check_mount_test_dev run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file bs=1M count=1 run_check $SUDO_HELPER "$TOP/btrfs" filesystem usage -T "$TEST_MNT" diff --git a/tests/mkfs-tests/032-zoned-reset/test.sh b/tests/mkfs-tests/032-zoned-reset/test.sh index 2aedb14abb..f899c3912d 100755 --- a/tests/mkfs-tests/032-zoned-reset/test.sh +++ b/tests/mkfs-tests/032-zoned-reset/test.sh @@ -13,29 +13,29 @@ prepare_nullbdevs TEST_DEV="${nullb_devs[1]}" last_zone_sector=$(( 4 * 31 * 1024 * 1024 / 512 )) # Write some data to the last zone -run_check $SUDO_HELPER dd if=/dev/urandom of="${TEST_DEV}" bs=1M count=4 seek=$(( 4 * 31 )) +run_check $SUDO_HELPER dd if=/dev/urandom of="$TEST_DEV" bs=1M count=4 seek=$(( 4 * 31 )) # Use single as it's supported on more kernels run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -m single -d single "${TEST_DEV}" # Check if the lat zone is empty -run_check_stdout $SUDO_HELPER blkzone report -o ${last_zone_sector} -c 1 "${TEST_DEV}" | grep -Fq '(em)' +run_check_stdout $SUDO_HELPER blkzone report -o "$last_zone_sector" -c 1 "$TEST_DEV" | grep -Fq '(em)' if [ $? != 0 ]; then _fail "last zone is not empty" fi # Write some data to the last zone -run_check $SUDO_HELPER dd if=/dev/urandom of="${TEST_DEV}" bs=1M count=1 seek=$(( 4 * 31 )) +run_check $SUDO_HELPER dd if=/dev/urandom of="$TEST_DEV" bs=1M count=1 seek=$(( 4 * 31 )) # Create a FS excluding the last zone -run_mayfail $SUDO_HELPER "$TOP/mkfs.btrfs" -f -b $(( 4 * 31 ))M -m single -d single "${TEST_DEV}" +run_mayfail $SUDO_HELPER "$TOP/mkfs.btrfs" -f -b $(( 4 * 31 ))M -m single -d single "$TEST_DEV" if [ $? == 0 ]; then _fail "mkfs.btrfs should detect active zone outside of FS range" fi # Fill the last zone to finish it -run_check $SUDO_HELPER dd if=/dev/urandom of="${TEST_DEV}" bs=1M count=3 seek=$(( 4 * 31 + 1 )) +run_check $SUDO_HELPER dd if=/dev/urandom of="$TEST_DEV" bs=1M count=3 seek=$(( 4 * 31 + 1 )) # Create a FS excluding the last zone -run_mayfail $SUDO_HELPER "$TOP/mkfs.btrfs" -f -b $(( 4 * 31 ))M -m single -d single "${TEST_DEV}" +run_mayfail $SUDO_HELPER "$TOP/mkfs.btrfs" -f -b $(( 4 * 31 ))M -m single -d single "$TEST_DEV" # Check if the lat zone is not empty -run_check_stdout $SUDO_HELPER blkzone report -o ${last_zone_sector} -c 1 "${TEST_DEV}" | grep -Fq '(em)' +run_check_stdout $SUDO_HELPER blkzone report -o "$last_zone_sector" -c 1 "$TEST_DEV" | grep -Fq '(em)' if [ $? == 0 ]; then _fail "last zone is empty" fi From 8e687f96ec0027663456d39d0dd930bfb7555728 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 01:56:14 +0200 Subject: [PATCH 07/35] btrfs-progs: completion: update mkswapfile, tree-stats and map-swapfile All the three commands need a file, update the completion rules. Signed-off-by: David Sterba --- btrfs-completion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/btrfs-completion b/btrfs-completion index 1d224087f9..3ec68e1bfd 100644 --- a/btrfs-completion +++ b/btrfs-completion @@ -103,7 +103,7 @@ _btrfs() case $cmd in filesystem) case $prev in - du|defragment) + du|defragment|mkswapfile) _filedir return 0 ;; @@ -128,7 +128,7 @@ _btrfs() _btrfs_mnts return 0 ;; - dump-tree|dump-super|rootid|inode-resolve) + dump-tree|dump-super|rootid|inode-resolve|tree-stats|map-swapfile) _filedir return 0 ;; From 8640307239b0603dc5dcdfa00343cd5b88a1108c Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 04:15:11 +0200 Subject: [PATCH 08/35] btrfs-progs: ci: add RockyLinux 9 image There's no Centos 9, use one of it's replacemnts for to continue the familiy of LTS distros. No exceptions for feature support (zoned, udev). Signed-off-by: David Sterba --- ci/ci-build-rockylinux9 | 34 +++++++++++++++++ ci/images/ci-rockylinux-9-x86_64/Dockerfile | 37 +++++++++++++++++++ ci/images/ci-rockylinux-9-x86_64/docker-build | 1 + ci/images/ci-rockylinux-9-x86_64/docker-run | 1 + ci/images/ci-rockylinux-9-x86_64/run-tests | 13 +++++++ ci/images/ci-rockylinux-9-x86_64/test-build | 23 ++++++++++++ 6 files changed, 109 insertions(+) create mode 100755 ci/ci-build-rockylinux9 create mode 100644 ci/images/ci-rockylinux-9-x86_64/Dockerfile create mode 120000 ci/images/ci-rockylinux-9-x86_64/docker-build create mode 120000 ci/images/ci-rockylinux-9-x86_64/docker-run create mode 100755 ci/images/ci-rockylinux-9-x86_64/run-tests create mode 100755 ci/images/ci-rockylinux-9-x86_64/test-build diff --git a/ci/ci-build-rockylinux9 b/ci/ci-build-rockylinux9 new file mode 100755 index 0000000000..74f242c725 --- /dev/null +++ b/ci/ci-build-rockylinux9 @@ -0,0 +1,34 @@ +#!/bin/sh +# Usage: $0 [branch] [configure options] +# Create source tarball from HEAD or given branch and build it in RockyLinux 9 CI +# environment. Configure options follow branch name that can be empty. + +HERE=`pwd` +if [ -f "configure.ac" ]; then + SOURCEDIR=`pwd` +elif [ -f "../configure.ac" ]; then + cd .. + SOURCEDIR=`pwd` +else + echo "ERROR: cannot determine source directory from `pwd`" + exit 1 +fi + +CIIMAGEDIR=ci/images/ci-rockylinux-9-x86_64 +BRANCH=${1:-HEAD} +if [ "$#" -ge 1 ]; then + shift +fi +HASH=$(git log -1 --format='%h %s' "$BRANCH") + +echo "CI: Generate archive from $BRANCH ($HASH)" +git archive --prefix=btrfs-progs-devel/ -o devel.tar "$BRANCH" +echo "$BRANCH $HASH" > GITCOMMIT +tar uvf devel.tar GITCOMMIT +#rm GITCOMMIT +gzip --force --best devel.tar + +cd "$CIIMAGEDIR" +cp "$SOURCEDIR/devel.tar.gz" . +./docker-build +./docker-run -- ./test-build devel --disable-documentation "$@" diff --git a/ci/images/ci-rockylinux-9-x86_64/Dockerfile b/ci/images/ci-rockylinux-9-x86_64/Dockerfile new file mode 100644 index 0000000000..7c1651caf4 --- /dev/null +++ b/ci/images/ci-rockylinux-9-x86_64/Dockerfile @@ -0,0 +1,37 @@ +FROM rockylinux:9 + +WORKDIR /tmp + +RUN rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org +RUN yum -y install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm +RUN yum -y install epel-release + +RUN yum -y install autoconf automake pkg-config +RUN yum -y install libattr-devel libblkid-devel libuuid-devel +RUN yum -y install e2fsprogs-libs e2fsprogs-devel +RUN yum -y install zlib-devel lzo-devel libzstd-devel zstd +RUN yum -y install make gcc tar gzip clang +RUN yum -y install python3 python3-devel python3-setuptools + +# For downloading fresh sources +RUN yum -y install wget + +# For running tests +RUN yum -y install util-linux e2fsprogs findutils grep +RUN yum -y install udev systemd-devel device-mapper acl attr xz + +RUN yum -y install libsodium-devel + +# For debugging +RUN yum -y install less vim + +COPY ./test-build . +COPY ./run-tests . +COPY ./devel.tar.gz . + +CMD ./test-build devel --disable-documentation + +# Continue with: +# cd /tmp +# (see CMD above) +# ./run-tests /tmp/btrfs-progs-devel diff --git a/ci/images/ci-rockylinux-9-x86_64/docker-build b/ci/images/ci-rockylinux-9-x86_64/docker-build new file mode 120000 index 0000000000..622545a7c1 --- /dev/null +++ b/ci/images/ci-rockylinux-9-x86_64/docker-build @@ -0,0 +1 @@ +../docker-build \ No newline at end of file diff --git a/ci/images/ci-rockylinux-9-x86_64/docker-run b/ci/images/ci-rockylinux-9-x86_64/docker-run new file mode 120000 index 0000000000..0903331186 --- /dev/null +++ b/ci/images/ci-rockylinux-9-x86_64/docker-run @@ -0,0 +1 @@ +../docker-run \ No newline at end of file diff --git a/ci/images/ci-rockylinux-9-x86_64/run-tests b/ci/images/ci-rockylinux-9-x86_64/run-tests new file mode 100755 index 0000000000..f2a1524252 --- /dev/null +++ b/ci/images/ci-rockylinux-9-x86_64/run-tests @@ -0,0 +1,13 @@ +#!/bin/sh -x + +where="$1" + +cd "$where" || { echo "ERROR: $1 not found"; exit 1; } + +make TEST_LOG=dump test-cli +make TEST_LOG=dump test-mkfs +make TEST_LOG=dump test-check +make TEST_LOG=dump test-check-lowmem +make TEST_LOG=dump test-misc +make TEST_LOG=dump test-convert +make TEST_LOG=dump test-fuzz diff --git a/ci/images/ci-rockylinux-9-x86_64/test-build b/ci/images/ci-rockylinux-9-x86_64/test-build new file mode 100755 index 0000000000..54559461bb --- /dev/null +++ b/ci/images/ci-rockylinux-9-x86_64/test-build @@ -0,0 +1,23 @@ +#!/bin/sh +# usage: $0 [branch name] [configure parameters] + +urlbase="https://github.com/kdave/btrfs-progs/archive" +branch=${1:-devel} +fnbase="${branch/\//-}" +fname="${fnbase}.tar.gz" +url="${urlbase}/${branch}.tar.gz" + +shift + +echo "btrfs-progs build test of branch ${branch}" +cd /tmp +if [ -f "$fname" -a -s "$fname" ]; then + echo "Found local file $fname, not downloading" +else + echo "Missing or empty tar, downloading branch $branch from git" + rm -- "$fname" + wget "$url" -O "$fname" +fi +tar xf "$fname" +cd "btrfs-progs-$fnbase" +ci/build-default "$@" From 0f2b1cc8521d2ca390ba95d5fe1e6d7fe01e3878 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 04:37:42 +0200 Subject: [PATCH 09/35] btrfs-progs: ci: enable RockyLinux 9 build tests Signed-off-by: David Sterba --- .github/workflows/ci-build-test-fast.yml | 6 ++++++ .github/workflows/ci-build-test.yml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/ci-build-test-fast.yml b/.github/workflows/ci-build-test-fast.yml index 1be2455544..e20367417a 100644 --- a/.github/workflows/ci-build-test-fast.yml +++ b/.github/workflows/ci-build-test-fast.yml @@ -24,6 +24,12 @@ jobs: steps: - name: CI Centos8 run: sudo docker run kdave/ci-centos-8-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation --disable-zoned --disable-libudev + check-rockylinux9: + name: CI RockyLinux 9 + runs-on: ubuntu-24.04 + steps: + - name: CI RockyLinux 9 + run: sudo docker run kdave/ci-rockylinux-9-x86_64 ./test-build $GITHUB_REF_NAME --disable-documentation check-leap153: name: CI Leap 15.3 runs-on: ubuntu-24.04 diff --git a/.github/workflows/ci-build-test.yml b/.github/workflows/ci-build-test.yml index c15d64413b..ba5175529c 100644 --- a/.github/workflows/ci-build-test.yml +++ b/.github/workflows/ci-build-test.yml @@ -24,6 +24,13 @@ jobs: - uses: actions/checkout@v4 - name: CI Centos8 run: ci/ci-build-centos8 + check-rockylinux9: + name: CI RockyLinux 9 + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: CI RockyLinux 9 + run: ci/ci-build-rockylinux9 check-leap153: name: CI Leap 15.3 runs-on: ubuntu-24.04 From 760e879b4a1021d42777b85c50a7fc998ed50305 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 21:17:14 +0200 Subject: [PATCH 10/35] btrfs-progs: tests: fix cleanup step in check/037 The test check/037-freespacetree-repair sometimes fails (does not in the CI though) that the last unmount does not work due to -EBUSY. This depends on timing, but what is obviously wrong is the 'rm' before umount. The quoting of "*" does not remove all the files before fallocate but is otherwise silent due to '-f'. Add the run_check so it's recorded in the logs taht it happens and alos add sudo so it can actually work. This makes the test work more reliably. Signed-off-by: David Sterba --- tests/fsck-tests/037-freespacetree-repair/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fsck-tests/037-freespacetree-repair/test.sh b/tests/fsck-tests/037-freespacetree-repair/test.sh index e615fd5e6e..66d2206d73 100755 --- a/tests/fsck-tests/037-freespacetree-repair/test.sh +++ b/tests/fsck-tests/037-freespacetree-repair/test.sh @@ -87,7 +87,7 @@ fi # change the freespace such that we now have at least one free_space_extent # object run_check_mount_test_dev -rm -rf "$TEST_MNT/file.*" +run_check $SUDO_HELPER rm -rf "$TEST_MNT/"file.* run_check $SUDO_HELPER fallocate -l 50m "$TEST_MNT/file" run_check_umount_test_dev From c1882f9e300b5a932716bd9ac66b45760653d696 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Jun 2024 09:58:48 +0930 Subject: [PATCH 11/35] btrfs-progs: tests: always wait for subvolume cleaning before clear-stale [BUG] There is a random chance that misc/055 would fail with stale qgroups detected. [CAUSE] Commit 82f7d6c1d711 ("btrfs-progs: qgroup: handle stale qgroup deletion more accurately") changed the behavior of "btrfs qgroup clear-stale" that it will no longer try to remove qgroup under deletion. And the test case itself relies on the return value of "btrfs qgroup clear-stale" to do the extra wait for subvolume deletion. This means after that commit, the test case would skip the wait if there is a subvolume waiting for cleanup, resulting in a race window where the subvolume can be dropped and become stale, and eventually trigger the stale qgroup detection and cause false alerts. [FIX] Fix the test case by always wait for the subvolume to be dropped, so that later "btrfs qgroup clear-stale" can properly remove all stale qgroups. Pull-request: #814 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- tests/misc-tests/055-qgroup-clear-stale/test.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/misc-tests/055-qgroup-clear-stale/test.sh b/tests/misc-tests/055-qgroup-clear-stale/test.sh index 3e029885a0..b0edf869b4 100755 --- a/tests/misc-tests/055-qgroup-clear-stale/test.sh +++ b/tests/misc-tests/055-qgroup-clear-stale/test.sh @@ -18,14 +18,15 @@ run_check $SUDO_HELPER "$TOP/btrfs" qgroup show "$TEST_MNT" rootid=$(run_check_stdout "$TOP/btrfs" inspect-internal rootid "$TEST_MNT/subv1") run_check $SUDO_HELPER "$TOP/btrfs" subvolume delete "$TEST_MNT/subv1" run_check $SUDO_HELPER "$TOP/btrfs" qgroup show "$TEST_MNT" -# Subvolume deletion could be slow and won't let the qgroup to be cleaned, -# shown as ''. -if ! run_mayfail $SUDO_HELPER "$TOP/btrfs" qgroup clear-stale "$TEST_MNT"; then - run_check $SUDO_HELPER "$TOP/btrfs" subvolume sync "$TEST_MNT" "$rootid" - run_check $SUDO_HELPER "$TOP/btrfs" qgroup show "$TEST_MNT" - # After cleaning the subvolume it must pass - run_check $SUDO_HELPER "$TOP/btrfs" qgroup clear-stale "$TEST_MNT" -fi + +# Subvolume under deletion won't be deleted and "btrfs qgroup clear-stale" +# would detect it and not report an error. +# So here we have to wait for the subvolume deletion. +run_check $SUDO_HELPER "$TOP/btrfs" subvolume sync "$TEST_MNT" "$rootid" +run_check $SUDO_HELPER "$TOP/btrfs" qgroup show "$TEST_MNT" +# After cleaning the subvolume it must pass +run_check $SUDO_HELPER "$TOP/btrfs" qgroup clear-stale "$TEST_MNT" + list=$(run_check_stdout $SUDO_HELPER "$TOP/btrfs" qgroup show "$TEST_MNT") if echo "$list" | grep -q "0/$rootid"; then _fail "stale qgroups not deleted" From 9cfa509133f503e13bbc817a0c2da57dbc05370e Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Jun 2024 10:30:24 +0930 Subject: [PATCH 12/35] btrfs-progs: tests: dump dmesg if a mount fails [BUG] There is a random failure for misc/028 in github CI, but unable to reproduce locally. [EXTRA DUMP] The failure is at mount time, which is mostly out of our control (kernel is provided by the CI image), but at least we can do extra kernel dump if a mount fails. This dump is done inside run_check() where if we detects there is a "mount" word inside the command string. The detection is not the best, it may detect other commands that contains the word "mount", but it should be good enough to detect real mount failure. Pull-request: #815 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- tests/common | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/common b/tests/common index 30a3201e50..449b6d654a 100644 --- a/tests/common +++ b/tests/common @@ -220,7 +220,13 @@ run_check() echo "====== RUN CHECK ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD: ${cmd_array[@]}" > /dev/tty; fi - "${cmd_array[@]}" >> "$RESULTS" 2>&1 || _fail "failed: ${cmd_array[@]}" + "${cmd_array[@]}" >> "$RESULTS" 2>&1 + if [ "$?" -ne 0 ]; then + if cat "${cmd_array[@]}" | grep -q mount; then + dmesg | tail -n 15 >> "$RESULTS" + fi + _fail "failed: ${cmd_array[@]}" + fi } # same as run_check but the stderr+stdout output is duplicated on stdout and From cc2230e41224a163b258250ec3845c2b7abb9902 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Jun 2024 08:00:32 +0930 Subject: [PATCH 13/35] btrfs-progs: convert: remove raid-stripe-tree support The raid-stripe-tree (RST) feature is for zoned devices to support extra data profiles, and is not yet a stable feature (still requires CONFIG_BTRFS_DEBUG enabled kernel to support it). Furthermore the supported filesystems (ext*, reiserfs and ntfs) don't even support zoned devices, and even we force RST support for btrfs-convert, we would only create an empty tree for RST, as btrfs convert would only result SINGLE data profile with SINGLE/DUP metadata profile, neither needs RST at all. Enabling RST for btrfs-convert would only cause problems for false test failures as we incorrectly allow RST feature for btrfs-convert. Fixes the problem by removing raid-stripe-tree support from btrfs-convert and remove the test cases support for RST. This patch is mostly reverting commit 346a3819237b ("btrfs-progs: convert: add raid-stripe-tree to allowed features"), but keeps the test infrastructure to support bgt features for convert. Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- common/fsfeatures.h | 3 +-- tests/convert-tests/001-ext2-basic/test.sh | 2 +- tests/convert-tests/003-ext4-basic/test.sh | 2 +- tests/convert-tests/005-delete-all-rollback/test.sh | 2 +- tests/convert-tests/010-reiserfs-basic/test.sh | 2 +- tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh | 2 +- tests/convert-tests/024-ntfs-basic/test.sh | 2 +- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/common/fsfeatures.h b/common/fsfeatures.h index 5b241bdf91..f636171f6b 100644 --- a/common/fsfeatures.h +++ b/common/fsfeatures.h @@ -68,8 +68,7 @@ static const struct btrfs_mkfs_features btrfs_convert_allowed_features = { BTRFS_FEATURE_INCOMPAT_BIG_METADATA | BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | - BTRFS_FEATURE_INCOMPAT_NO_HOLES | - BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE, + BTRFS_FEATURE_INCOMPAT_NO_HOLES, .runtime_flags = BTRFS_FEATURE_RUNTIME_QUOTA, }; diff --git a/tests/convert-tests/001-ext2-basic/test.sh b/tests/convert-tests/001-ext2-basic/test.sh index 461ff4a5f1..d905ed4aee 100755 --- a/tests/convert-tests/001-ext2-basic/test.sh +++ b/tests/convert-tests/001-ext2-basic/test.sh @@ -12,7 +12,7 @@ check_kernel_support_acl # Iterate over defaults and options that are not tied to hardware capabilities # or number of devices -for feature in '' 'block-group-tree' 'raid-stripe-tree'; do +for feature in '' 'block-group-tree'; do convert_test ext2 "$feature" "ext2 4k nodesize" 4096 mke2fs -b 4096 convert_test ext2 "$feature" "ext2 16k nodesize" 16384 mke2fs -b 4096 convert_test ext2 "$feature" "ext2 64k nodesize" 65536 mke2fs -b 4096 diff --git a/tests/convert-tests/003-ext4-basic/test.sh b/tests/convert-tests/003-ext4-basic/test.sh index 14637fc852..83a2cf1242 100755 --- a/tests/convert-tests/003-ext4-basic/test.sh +++ b/tests/convert-tests/003-ext4-basic/test.sh @@ -12,7 +12,7 @@ check_kernel_support_acl # Iterate over defaults and options that are not tied to hardware capabilities # or number of devices -for feature in '' 'block-group-tree' 'raid-stripe-tree' ; do +for feature in '' 'block-group-tree'; do convert_test ext4 "$feature" "ext4 4k nodesize" 4096 mke2fs -t ext4 -b 4096 convert_test ext4 "$feature" "ext4 16k nodesize" 16384 mke2fs -t ext4 -b 4096 convert_test ext4 "$feature" "ext4 64k nodesize" 65536 mke2fs -t ext4 -b 4096 diff --git a/tests/convert-tests/005-delete-all-rollback/test.sh b/tests/convert-tests/005-delete-all-rollback/test.sh index d0be331ad4..58f4a6fda7 100755 --- a/tests/convert-tests/005-delete-all-rollback/test.sh +++ b/tests/convert-tests/005-delete-all-rollback/test.sh @@ -68,7 +68,7 @@ do_test() { # Iterate over defaults and options that are not tied to hardware capabilities # or number of devices -for feature in '' 'block-group-tree' 'raid-stripe-tree' ; do +for feature in '' 'block-group-tree'; do do_test "$feature" "ext4 4k nodesize" 4096 mke2fs -t ext4 -b 4096 do_test "$feature" "ext4 16k nodesize" 16384 mke2fs -t ext4 -b 4096 do_test "$feature" "ext4 64k nodesize" 65536 mke2fs -t ext4 -b 4096 diff --git a/tests/convert-tests/010-reiserfs-basic/test.sh b/tests/convert-tests/010-reiserfs-basic/test.sh index 6ab02b31d1..67e818b982 100755 --- a/tests/convert-tests/010-reiserfs-basic/test.sh +++ b/tests/convert-tests/010-reiserfs-basic/test.sh @@ -15,7 +15,7 @@ prepare_test_dev # Iterate over defaults and options that are not tied to hardware capabilities # or number of devices -for feature in '' 'block-group-tree' 'raid-stripe-tree'; do +for feature in '' 'block-group-tree'; do convert_test reiserfs "$feature" "reiserfs 4k nodesize" 4096 mkreiserfs -b 4096 convert_test reiserfs "$feature" "reiserfs 16k nodesize" 16384 mkreiserfs -b 4096 convert_test reiserfs "$feature" "reiserfs 64k nodesize" 65536 mkreiserfs -b 4096 diff --git a/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh b/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh index 688613c2c6..d874422659 100755 --- a/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh +++ b/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh @@ -70,7 +70,7 @@ do_test() { # Iterate over defaults and options that are not tied to hardware capabilities # or number of devices -for feature in '' 'block-group-tree' 'raid-stripe-tree'; do +for feature in '' 'block-group-tree'; do do_test "$feature" "reiserfs 4k nodesize" 4096 mkreiserfs -b 4096 do_test "$feature" "reiserfs 16k nodesize" 16384 mkreiserfs -b 4096 do_test "$feature" "reiserfs 64k nodesize" 65536 mkreiserfs -b 4096 diff --git a/tests/convert-tests/024-ntfs-basic/test.sh b/tests/convert-tests/024-ntfs-basic/test.sh index a93f600706..b88dbe29c3 100755 --- a/tests/convert-tests/024-ntfs-basic/test.sh +++ b/tests/convert-tests/024-ntfs-basic/test.sh @@ -17,6 +17,6 @@ prepare_test_dev # Iterate over defaults and options that are not tied to hardware capabilities # or number of devices. Test only 4K block size as minimum. -for feature in '' 'block-group-tree' 'raid-stripe-tree'; do +for feature in '' 'block-group-tree'; do convert_test ntfs "$feature" "ntfs 4k nodesize" 4096 mkfs.ntfs -s 4096 done From 2781f669e34a0f532f6ec29de62ff891228c0b46 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Jun 2024 08:36:41 +0930 Subject: [PATCH 14/35] btrfs-progs: tests: hide RST related mkfs tests behind experimental builds Currently we unconditionally run mkfs/029 and mkfs/030 as we export RST feature through mkfs.btrfs as supported features. But considering the mkfs.btrfs features don't match kernel support (only a BTRFS_DEBUG kernel can support that), we're going to hide RST behind experimental builds. In that case RST related tests cases also need to be behind experimental builds as regular builds of mkfs.btrfs will no longer support RST feature. Introduce two helpers: - _test_config() To verify if certain config is set to 1 - check_experimental_build() A wrapper of "_test_config EXPERIMENTAL", and skip the test if btrfs-progs has no experimental features. So that test cases can be skipped for non-experimental builds. Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- tests/common | 23 +++++++++++++++++++ tests/mkfs-tests/029-raid-stripe-tree/test.sh | 1 + tests/mkfs-tests/030-zoned-rst/test.sh | 1 + 3 files changed, 25 insertions(+) diff --git a/tests/common b/tests/common index 449b6d654a..310f5daa51 100644 --- a/tests/common +++ b/tests/common @@ -6,6 +6,22 @@ # Temporary array for building the final command, injecting arguments as needed declare -a cmd_array +# Check if a given option in config.h is set to 1 +# $1: config option name +_test_config() +{ + local feature="$1" + + if [ ! -f "$TOP/include/config.h" ]; then + echo "include/config.h not exists" + exit 1 + fi + if grep -q "$feature.*1" "${TOP}/include/config.h"; then + return 0 + fi + return 1 +} + # assert that argument is not empty and is an existing path (file or directory) _assert_path() { @@ -376,6 +392,13 @@ run_mustfail_stdout() fi } +check_experimental_build() +{ + if ! _test_config "EXPERIMENTAL"; then + _not_run "This test requires experimental build" + fi +} + check_prereq() { # Internal tools for testing, not shipped with the package diff --git a/tests/mkfs-tests/029-raid-stripe-tree/test.sh b/tests/mkfs-tests/029-raid-stripe-tree/test.sh index c3edd36d87..f200d068fe 100755 --- a/tests/mkfs-tests/029-raid-stripe-tree/test.sh +++ b/tests/mkfs-tests/029-raid-stripe-tree/test.sh @@ -3,6 +3,7 @@ source "$TEST_TOP/common" || exit +check_experimental_build check_prereq mkfs.btrfs check_prereq btrfs diff --git a/tests/mkfs-tests/030-zoned-rst/test.sh b/tests/mkfs-tests/030-zoned-rst/test.sh index b1c696c96e..b03c691f1b 100755 --- a/tests/mkfs-tests/030-zoned-rst/test.sh +++ b/tests/mkfs-tests/030-zoned-rst/test.sh @@ -3,6 +3,7 @@ source "$TEST_TOP/common" || exit +check_experimental_build setup_root_helper setup_nullbdevs 4 128 4 prepare_nullbdevs From c600fcca81ec4d971892a8112dfb094f11d62adf Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Jun 2024 08:51:50 +0930 Subject: [PATCH 15/35] btrfs-progs: move RST back to experimental Currently raid-stripe-tree feature is still experimental as it requires a BTRFS_DEBUG kernel to recognize it. To avoid confusion move it back to experimental so regular users won't incorrectly set it. And since RST is no longer supported by default, also change the RST profile detection so that for non-experimental build we won't enable RST according to the data profiles. Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- common/fsfeatures.c | 2 ++ mkfs/main.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/common/fsfeatures.c b/common/fsfeatures.c index c5e81629cc..7aaddab6a1 100644 --- a/common/fsfeatures.c +++ b/common/fsfeatures.c @@ -222,6 +222,7 @@ static const struct btrfs_feature mkfs_features[] = { VERSION_NULL(default), .desc = "block group tree, more efficient block group tracking to reduce mount time" }, +#if EXPERIMENTAL { .name = "rst", .incompat_flag = BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE, @@ -238,6 +239,7 @@ static const struct btrfs_feature mkfs_features[] = { VERSION_NULL(default), .desc = "raid stripe tree, enhanced file extent tracking" }, +#endif { .name = "squota", .incompat_flag = BTRFS_FEATURE_INCOMPAT_SIMPLE_QUOTA, diff --git a/mkfs/main.c b/mkfs/main.c index 0b9d1d7866..8d0924ea3b 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1693,7 +1693,9 @@ int BOX_MAIN(mkfs)(int argc, char **argv) case BTRFS_BLOCK_GROUP_RAID1C4: case BTRFS_BLOCK_GROUP_RAID0: case BTRFS_BLOCK_GROUP_RAID10: +#if EXPERIMENTAL features.incompat_flags |= BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE; +#endif break; default: break; From 12358fa0615b6943dc5ecc935d4ec4d2cd42bbc7 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Jun 2024 09:05:01 +0930 Subject: [PATCH 16/35] btrfs-progs: mkfs-tests: ensure regular builds won't enable rst feature This test case will check the following behviour: - Regular zoned supported data/metadata profiles Should success - RST with zoned feature Should fail for non-experimental builds - zoned with data profiles that only RST supports Should fail for non-experimental builds. Meanwhile for experimental builds it should success and enable RST feature automatically (verified in mkfs/030) Pull-request: #813 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- tests/common | 7 ++++ tests/mkfs-tests/033-zoned-reject-rst/test.sh | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100755 tests/mkfs-tests/033-zoned-reject-rst/test.sh diff --git a/tests/common b/tests/common index 310f5daa51..e9b973e448 100644 --- a/tests/common +++ b/tests/common @@ -399,6 +399,13 @@ check_experimental_build() fi } +check_regular_build() +{ + if _test_config "EXPERIMENTAL"; then + _not_run "This test requires non-experimental build" + fi +} + check_prereq() { # Internal tools for testing, not shipped with the package diff --git a/tests/mkfs-tests/033-zoned-reject-rst/test.sh b/tests/mkfs-tests/033-zoned-reject-rst/test.sh new file mode 100755 index 0000000000..c9c5c123a5 --- /dev/null +++ b/tests/mkfs-tests/033-zoned-reject-rst/test.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Verify mkfs for all currently supported profiles of zoned + raid-stripe-tree + +source "$TEST_TOP/common" || exit + +check_regular_build +setup_root_helper +setup_nullbdevs 4 128 4 +prepare_nullbdevs +TEST_DEV=${nullb_devs[1]} + +profiles="dup raid1 raid1c3 raid1c4 raid10" + +# The existing supported profiles. +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -O zoned -d single -m single "${nullb_devs[@]}" +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -O zoned -d single -m DUP "${nullb_devs[@]}" + +# RST feature is rejected +run_mustfail "RST feature created" \ + $SUDO_HELPER "$TOP/mkfs.btrfs" -f -O zoned,raid-stripe-tree -d single -m DUP "${nullb_devs[@]}" + +for dprofile in $profiles; do + # Make sure additional data profiles won't enable RST for non-experimental build + run_mustfail "unsupported profile created" \ + $SUDO_HELPER "$TOP/mkfs.btrfs" -f -O zoned -d "$dprofile" -m DUP "${nullb_devs[@]}" +done + +# The old unsupported profiles should fail no matter if experimental build is enabled or not. +run_mustfail "unsupported profile raid56 created" \ + $SUDO_HELPER "$TOP/mkfs.btrfs" -f -O zoned -d raid5 -m raid5 "${nullb_devs[@]}" +run_mustfail "unsupported profile raid56 created" \ + $SUDO_HELPER "$TOP/mkfs.btrfs" -f -O zoned -d raid6 -m raid6 "${nullb_devs[@]}" + +cleanup_nullbdevs From 95062752421ad71b8f39ec6c488b0d7a25e763b7 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 20:24:02 +0200 Subject: [PATCH 17/35] btrfs-progs: build: add kernel-shared nested directories to tags The tags can't find structures defined in kernel-shared/uapi/*.h due to missing path. Signed-off-by: David Sterba --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 49fb8d9bf2..79e9b5b3ab 100644 --- a/Makefile +++ b/Makefile @@ -878,6 +878,7 @@ tags: FORCE @echo " [TAGS] $(TAGS_CMD)" $(Q)$(TAGS_CMD) *.[ch] image/*.[ch] convert/*.[ch] mkfs/*.[ch] \ check/*.[ch] kernel-lib/*.[ch] kernel-shared/*.[ch] \ + kernel-shared/*/*.[ch] \ cmds/*.[ch] common/*.[ch] tune/*.[ch] \ libbtrfsutil/*.[ch] From 22b3246ef9c3b83cff49c10ec0b398cd834b4872 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 20:28:13 +0200 Subject: [PATCH 18/35] btrfs-progs: copy entire label buffer to target buffers The label is of a fixed size 256 bytes and expects the zero terminator. Using __strncpy_null is correct as it makes sure there's always the zero termination but the argument passed in skips the last character. Signed-off-by: David Sterba --- common/filesystem-utils.c | 8 ++++---- convert/common.c | 2 +- convert/main.c | 4 ++-- mkfs/common.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/filesystem-utils.c b/common/filesystem-utils.c index 66269ad92f..8dee27aab3 100644 --- a/common/filesystem-utils.c +++ b/common/filesystem-utils.c @@ -102,7 +102,7 @@ static int set_label_unmounted(const char *dev, const char *label) error_msg(ERROR_MSG_START_TRANS, "set label"); return PTR_ERR(trans); } - __strncpy_null(root->fs_info->super_copy->label, label, BTRFS_LABEL_SIZE - 1); + __strncpy_null(root->fs_info->super_copy->label, label, BTRFS_LABEL_SIZE); btrfs_commit_transaction(trans, root); @@ -123,7 +123,7 @@ static int set_label_mounted(const char *mount_path, const char *labelp) } memset(label, 0, sizeof(label)); - __strncpy_null(label, labelp, BTRFS_LABEL_SIZE - 1); + __strncpy_null(label, labelp, BTRFS_LABEL_SIZE); if (ioctl(fd, BTRFS_IOC_SET_FSLABEL, label) < 0) { error("unable to set label of %s: %m", mount_path); close(fd); @@ -153,7 +153,7 @@ int get_label_unmounted(const char *dev, char *label) return -1; __strncpy_null(label, root->fs_info->super_copy->label, - BTRFS_LABEL_SIZE - 1); + BTRFS_LABEL_SIZE); /* Now we close it since we are done. */ close_ctree(root); @@ -187,7 +187,7 @@ int get_label_mounted(const char *mount_path, char *labelp) return ret; } - __strncpy_null(labelp, label, BTRFS_LABEL_SIZE - 1); + __strncpy_null(labelp, label, BTRFS_LABEL_SIZE); close(fd); return 0; } diff --git a/convert/common.c b/convert/common.c index 508697e2b1..d0d3e087c7 100644 --- a/convert/common.c +++ b/convert/common.c @@ -148,7 +148,7 @@ static int setup_temp_super(int fd, struct btrfs_mkfs_config *cfg, btrfs_set_super_cache_generation(&super, -1); btrfs_set_super_incompat_flags(&super, cfg->features.incompat_flags); if (cfg->label) - __strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE - 1); + __strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE); /* Sys chunk array will be re-initialized at chunk tree init time */ super.sys_chunk_array_size = 0; diff --git a/convert/main.c b/convert/main.c index 139fc15be1..3d2fd61a71 100644 --- a/convert/main.c +++ b/convert/main.c @@ -1323,7 +1323,7 @@ static int do_convert(const char *devname, u32 convert_flags, u32 nodesize, memset(root->fs_info->super_copy->label, 0, BTRFS_LABEL_SIZE); if (convert_flags & CONVERT_FLAG_COPY_LABEL) { __strncpy_null(root->fs_info->super_copy->label, - cctx.label, BTRFS_LABEL_SIZE - 1); + cctx.label, BTRFS_LABEL_SIZE); printf("Copy label '%s'\n", root->fs_info->super_copy->label); } else if (convert_flags & CONVERT_FLAG_SET_LABEL) { strcpy(root->fs_info->super_copy->label, fslabel); @@ -1938,7 +1938,7 @@ int BOX_MAIN(convert)(int argc, char *argv[]) "label too long, trimmed to %d bytes", BTRFS_LABEL_SIZE - 1); } - __strncpy_null(fslabel, optarg, BTRFS_LABEL_SIZE - 1); + __strncpy_null(fslabel, optarg, BTRFS_LABEL_SIZE); break; case 'L': copylabel = CONVERT_FLAG_COPY_LABEL; diff --git a/mkfs/common.c b/mkfs/common.c index ec3b25b87f..ee0284bfd4 100644 --- a/mkfs/common.c +++ b/mkfs/common.c @@ -476,7 +476,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg) btrfs_set_super_nr_global_roots(&super, 1); if (cfg->label) - __strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE - 1); + __strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE); /* create the tree of root objects */ memset(buf->data, 0, cfg->nodesize); From 4168fa41cca087232f3bd64a36c9b63b3b780d6c Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 20:32:30 +0200 Subject: [PATCH 19/35] btrfs-progs: use explicit length parameters for string copy The macro strncpy_null uses sizeof the first argument for the length, but there are no checks and this works only for buffers with static length, i.e. not pointers. This is error prone. Use the open coded variant that makes the sizeof visible. Signed-off-by: David Sterba --- cmds/device.c | 10 +++++----- cmds/filesystem.c | 2 +- cmds/receive.c | 10 +++++----- cmds/subvolume.c | 9 ++++----- common/device-scan.c | 4 ++-- common/path-utils.c | 6 +++--- 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/cmds/device.c b/cmds/device.c index bff2bad917..09d3d54e67 100644 --- a/cmds/device.c +++ b/cmds/device.c @@ -169,7 +169,7 @@ static int cmd_device_add(const struct cmd_struct *cmd, } memset(&ioctl_args, 0, sizeof(ioctl_args)); - strncpy_null(ioctl_args.name, path); + __strncpy_null(ioctl_args.name, path, sizeof(ioctl_args.name)); res = ioctl(fdmnt, BTRFS_IOC_ADD_DEV, &ioctl_args); if (res < 0) { error("error adding device '%s': %m", path); @@ -287,7 +287,7 @@ static int _cmd_device_remove(const struct cmd_struct *cmd, } else if (strcmp(argv[i], "missing") == 0 || cancel || path_is_block_device(argv[i]) == 1) { - strncpy_null(argv2.name, argv[i]); + __strncpy_null(argv2.name, argv[i], sizeof(argv2.name)); } else { error("not a block device: %s", argv[i]); ret++; @@ -312,7 +312,7 @@ static int _cmd_device_remove(const struct cmd_struct *cmd, continue; } memset(&arg, 0, sizeof(arg)); - strncpy_null(arg.name, argv[i]); + __strncpy_null(arg.name, argv[i], sizeof(arg.name)); res = ioctl(fdmnt, BTRFS_IOC_RM_DEV, &arg); } @@ -396,7 +396,7 @@ static int btrfs_forget_devices(const char *path) memset(&args, 0, sizeof(args)); if (path) - strncpy_null(args.name, path); + __strncpy_null(args.name, path, sizeof(args.name)); ret = ioctl(fd, BTRFS_IOC_FORGET_DEV, &args); if (ret) ret = -errno; @@ -557,7 +557,7 @@ static int cmd_device_ready(const struct cmd_struct *cmd, int argc, char **argv) } memset(&args, 0, sizeof(args)); - strncpy_null(args.name, path); + __strncpy_null(args.name, path, sizeof(args.name)); ret = ioctl(fd, BTRFS_IOC_DEVICES_READY, &args); if (ret < 0) { error("unable to determine if device '%s' is ready for mount: %m", diff --git a/cmds/filesystem.c b/cmds/filesystem.c index c9930a02d8..13240e547f 100644 --- a/cmds/filesystem.c +++ b/cmds/filesystem.c @@ -1441,7 +1441,7 @@ static int cmd_filesystem_resize(const struct cmd_struct *cmd, memset(&args, 0, sizeof(args)); if (devid == (u64)-1) { /* Ok to copy the string verbatim. */ - strncpy_null(args.name, amount); + __strncpy_null(args.name, amount, sizeof(args.name)); } else { /* The implicit devid 1 needs to be adjusted. */ snprintf(args.name, sizeof(args.name) - 1, "%llu:%s", devid, amount); diff --git a/cmds/receive.c b/cmds/receive.c index e4430b077e..98760533d8 100644 --- a/cmds/receive.c +++ b/cmds/receive.c @@ -179,7 +179,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid, } if (*rctx->dest_dir_path == 0) { - strncpy_null(rctx->cur_subvol_path, path); + __strncpy_null(rctx->cur_subvol_path, path, sizeof(rctx->cur_subvol_path)); } else { ret = path_cat_out(rctx->cur_subvol_path, rctx->dest_dir_path, path); @@ -209,7 +209,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid, } memset(&args_v1, 0, sizeof(args_v1)); - strncpy_null(args_v1.name, path); + __strncpy_null(args_v1.name, path, sizeof(args_v1.name)); ret = ioctl(rctx->dest_dir_fd, BTRFS_IOC_SUBVOL_CREATE, &args_v1); if (ret < 0) { ret = -errno; @@ -249,7 +249,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, } if (*rctx->dest_dir_path == 0) { - strncpy_null(rctx->cur_subvol_path, path); + __strncpy_null(rctx->cur_subvol_path, path, sizeof(rctx->cur_subvol_path)); } else { ret = path_cat_out(rctx->cur_subvol_path, rctx->dest_dir_path, path); @@ -281,7 +281,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, } memset(&args_v2, 0, sizeof(args_v2)); - strncpy_null(args_v2.name, path); + __strncpy_null(args_v2.name, path, sizeof(args_v2.name)); parent_subvol = subvol_uuid_search(rctx->mnt_fd, 0, parent_uuid, parent_ctransid, NULL, @@ -663,7 +663,7 @@ static int open_inode_for_write(struct btrfs_receive *rctx, const char *path) error("cannot open %s: %m", path); goto out; } - strncpy_null(rctx->write_path, path); + __strncpy_null(rctx->write_path, path, sizeof(rctx->write_path)); out: return ret; diff --git a/cmds/subvolume.c b/cmds/subvolume.c index f77a6e0915..31d778e3ef 100644 --- a/cmds/subvolume.c +++ b/cmds/subvolume.c @@ -197,7 +197,7 @@ static int create_one_subvolume(const char *dst, struct btrfs_qgroup_inherit *in char dstdir_dup[PATH_MAX]; char *token; - strncpy_null(dstdir_dup, dstdir); + __strncpy_null(dstdir_dup, dstdir, sizeof(dstdir_dup)); if (dstdir_dup[0] == '/') strcat(p, "/"); @@ -233,7 +233,7 @@ static int create_one_subvolume(const char *dst, struct btrfs_qgroup_inherit *in struct btrfs_ioctl_vol_args_v2 args; memset(&args, 0, sizeof(args)); - strncpy_null(args.name, newname); + __strncpy_null(args.name, newname, sizeof(args.name)); args.flags |= BTRFS_SUBVOL_QGROUP_INHERIT; args.size = btrfs_qgroup_inherit_size(inherit); args.qgroup_inherit = inherit; @@ -243,8 +243,7 @@ static int create_one_subvolume(const char *dst, struct btrfs_qgroup_inherit *in struct btrfs_ioctl_vol_args args; memset(&args, 0, sizeof(args)); - strncpy_null(args.name, newname); - + __strncpy_null(args.name, newname, sizeof(args.name)); ret = ioctl(fddst, BTRFS_IOC_SUBVOL_CREATE, &args); } @@ -738,7 +737,7 @@ static int cmd_subvolume_snapshot(const struct cmd_struct *cmd, int argc, char * args.size = btrfs_qgroup_inherit_size(inherit); args.qgroup_inherit = inherit; } - strncpy_null(args.name, newname); + __strncpy_null(args.name, newname, sizeof(args.name)); res = ioctl(fddst, BTRFS_IOC_SNAP_CREATE_V2, &args); if (res < 0) { diff --git a/common/device-scan.c b/common/device-scan.c index bf6b6d575e..45b2530bb2 100644 --- a/common/device-scan.c +++ b/common/device-scan.c @@ -237,7 +237,7 @@ int btrfs_register_one_device(const char *fname) return -errno; } memset(&args, 0, sizeof(args)); - strncpy_null(args.name, fname); + __strncpy_null(args.name, fname, sizeof(args.name)); ret = ioctl(fd, BTRFS_IOC_SCAN_DEV, &args); if (ret < 0) { error("device scan failed on '%s': %m", fname); @@ -468,7 +468,7 @@ int btrfs_scan_devices(int verbose) if (!dev) continue; /* if we are here its definitely a btrfs disk*/ - strncpy_null(path, blkid_dev_devname(dev)); + __strncpy_null(path, blkid_dev_devname(dev), sizeof(path)); if (stat(path, &dev_stat) < 0) continue; diff --git a/common/path-utils.c b/common/path-utils.c index 494fe8288d..9f907c40fa 100644 --- a/common/path-utils.c +++ b/common/path-utils.c @@ -193,10 +193,10 @@ static int is_same_blk_file(const char* a, const char* b) char real_b[PATH_MAX]; if (!realpath(a, real_a)) - strncpy_null(real_a, a); + __strncpy_null(real_a, a, sizeof(real_a)); if (!realpath(b, real_b)) - strncpy_null(real_b, b); + __strncpy_null(real_b, b, sizeof(real_b)); /* Identical path? */ if (strcmp(real_a, real_b) == 0) @@ -403,7 +403,7 @@ int path_is_in_dir(const char *parent, const char *path) char *curr_dir = tmp; int ret; - strncpy_null(tmp, path); + __strncpy_null(tmp, path, sizeof(tmp)); while (strcmp(parent, curr_dir) != 0) { if (strcmp(curr_dir, "/") == 0) { From 6ed16f7b09a5d5d8150fef77e96395577ac1b3a3 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 20:44:25 +0200 Subject: [PATCH 20/35] btrfs-progs: rename and move __strncpy_null to string-utils Now that there's only __strncpy_null we can drop the underscore and move it to string-utils as it's a generic string function rather than something for paths. Signed-off-by: David Sterba --- cmds/device.c | 10 +++++----- cmds/filesystem.c | 2 +- cmds/receive.c | 10 +++++----- cmds/subvolume.c | 8 ++++---- common/device-scan.c | 5 +++-- common/filesystem-utils.c | 10 +++++----- common/path-utils.c | 29 +++++------------------------ common/path-utils.h | 5 ----- common/string-utils.c | 20 ++++++++++++++++++++ common/string-utils.h | 2 ++ convert/common.c | 3 ++- convert/main.c | 5 ++--- mkfs/common.c | 3 ++- 13 files changed, 56 insertions(+), 56 deletions(-) diff --git a/cmds/device.c b/cmds/device.c index 09d3d54e67..6a9a4db202 100644 --- a/cmds/device.c +++ b/cmds/device.c @@ -169,7 +169,7 @@ static int cmd_device_add(const struct cmd_struct *cmd, } memset(&ioctl_args, 0, sizeof(ioctl_args)); - __strncpy_null(ioctl_args.name, path, sizeof(ioctl_args.name)); + strncpy_null(ioctl_args.name, path, sizeof(ioctl_args.name)); res = ioctl(fdmnt, BTRFS_IOC_ADD_DEV, &ioctl_args); if (res < 0) { error("error adding device '%s': %m", path); @@ -287,7 +287,7 @@ static int _cmd_device_remove(const struct cmd_struct *cmd, } else if (strcmp(argv[i], "missing") == 0 || cancel || path_is_block_device(argv[i]) == 1) { - __strncpy_null(argv2.name, argv[i], sizeof(argv2.name)); + strncpy_null(argv2.name, argv[i], sizeof(argv2.name)); } else { error("not a block device: %s", argv[i]); ret++; @@ -312,7 +312,7 @@ static int _cmd_device_remove(const struct cmd_struct *cmd, continue; } memset(&arg, 0, sizeof(arg)); - __strncpy_null(arg.name, argv[i], sizeof(arg.name)); + strncpy_null(arg.name, argv[i], sizeof(arg.name)); res = ioctl(fdmnt, BTRFS_IOC_RM_DEV, &arg); } @@ -396,7 +396,7 @@ static int btrfs_forget_devices(const char *path) memset(&args, 0, sizeof(args)); if (path) - __strncpy_null(args.name, path, sizeof(args.name)); + strncpy_null(args.name, path, sizeof(args.name)); ret = ioctl(fd, BTRFS_IOC_FORGET_DEV, &args); if (ret) ret = -errno; @@ -557,7 +557,7 @@ static int cmd_device_ready(const struct cmd_struct *cmd, int argc, char **argv) } memset(&args, 0, sizeof(args)); - __strncpy_null(args.name, path, sizeof(args.name)); + strncpy_null(args.name, path, sizeof(args.name)); ret = ioctl(fd, BTRFS_IOC_DEVICES_READY, &args); if (ret < 0) { error("unable to determine if device '%s' is ready for mount: %m", diff --git a/cmds/filesystem.c b/cmds/filesystem.c index 13240e547f..291abeedf3 100644 --- a/cmds/filesystem.c +++ b/cmds/filesystem.c @@ -1441,7 +1441,7 @@ static int cmd_filesystem_resize(const struct cmd_struct *cmd, memset(&args, 0, sizeof(args)); if (devid == (u64)-1) { /* Ok to copy the string verbatim. */ - __strncpy_null(args.name, amount, sizeof(args.name)); + strncpy_null(args.name, amount, sizeof(args.name)); } else { /* The implicit devid 1 needs to be adjusted. */ snprintf(args.name, sizeof(args.name) - 1, "%llu:%s", devid, amount); diff --git a/cmds/receive.c b/cmds/receive.c index 98760533d8..412bc8afeb 100644 --- a/cmds/receive.c +++ b/cmds/receive.c @@ -179,7 +179,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid, } if (*rctx->dest_dir_path == 0) { - __strncpy_null(rctx->cur_subvol_path, path, sizeof(rctx->cur_subvol_path)); + strncpy_null(rctx->cur_subvol_path, path, sizeof(rctx->cur_subvol_path)); } else { ret = path_cat_out(rctx->cur_subvol_path, rctx->dest_dir_path, path); @@ -209,7 +209,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid, } memset(&args_v1, 0, sizeof(args_v1)); - __strncpy_null(args_v1.name, path, sizeof(args_v1.name)); + strncpy_null(args_v1.name, path, sizeof(args_v1.name)); ret = ioctl(rctx->dest_dir_fd, BTRFS_IOC_SUBVOL_CREATE, &args_v1); if (ret < 0) { ret = -errno; @@ -249,7 +249,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, } if (*rctx->dest_dir_path == 0) { - __strncpy_null(rctx->cur_subvol_path, path, sizeof(rctx->cur_subvol_path)); + strncpy_null(rctx->cur_subvol_path, path, sizeof(rctx->cur_subvol_path)); } else { ret = path_cat_out(rctx->cur_subvol_path, rctx->dest_dir_path, path); @@ -281,7 +281,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, } memset(&args_v2, 0, sizeof(args_v2)); - __strncpy_null(args_v2.name, path, sizeof(args_v2.name)); + strncpy_null(args_v2.name, path, sizeof(args_v2.name)); parent_subvol = subvol_uuid_search(rctx->mnt_fd, 0, parent_uuid, parent_ctransid, NULL, @@ -663,7 +663,7 @@ static int open_inode_for_write(struct btrfs_receive *rctx, const char *path) error("cannot open %s: %m", path); goto out; } - __strncpy_null(rctx->write_path, path, sizeof(rctx->write_path)); + strncpy_null(rctx->write_path, path, sizeof(rctx->write_path)); out: return ret; diff --git a/cmds/subvolume.c b/cmds/subvolume.c index 31d778e3ef..52bc88500e 100644 --- a/cmds/subvolume.c +++ b/cmds/subvolume.c @@ -197,7 +197,7 @@ static int create_one_subvolume(const char *dst, struct btrfs_qgroup_inherit *in char dstdir_dup[PATH_MAX]; char *token; - __strncpy_null(dstdir_dup, dstdir, sizeof(dstdir_dup)); + strncpy_null(dstdir_dup, dstdir, sizeof(dstdir_dup)); if (dstdir_dup[0] == '/') strcat(p, "/"); @@ -233,7 +233,7 @@ static int create_one_subvolume(const char *dst, struct btrfs_qgroup_inherit *in struct btrfs_ioctl_vol_args_v2 args; memset(&args, 0, sizeof(args)); - __strncpy_null(args.name, newname, sizeof(args.name)); + strncpy_null(args.name, newname, sizeof(args.name)); args.flags |= BTRFS_SUBVOL_QGROUP_INHERIT; args.size = btrfs_qgroup_inherit_size(inherit); args.qgroup_inherit = inherit; @@ -243,7 +243,7 @@ static int create_one_subvolume(const char *dst, struct btrfs_qgroup_inherit *in struct btrfs_ioctl_vol_args args; memset(&args, 0, sizeof(args)); - __strncpy_null(args.name, newname, sizeof(args.name)); + strncpy_null(args.name, newname, sizeof(args.name)); ret = ioctl(fddst, BTRFS_IOC_SUBVOL_CREATE, &args); } @@ -737,7 +737,7 @@ static int cmd_subvolume_snapshot(const struct cmd_struct *cmd, int argc, char * args.size = btrfs_qgroup_inherit_size(inherit); args.qgroup_inherit = inherit; } - __strncpy_null(args.name, newname, sizeof(args.name)); + strncpy_null(args.name, newname, sizeof(args.name)); res = ioctl(fddst, BTRFS_IOC_SNAP_CREATE_V2, &args); if (res < 0) { diff --git a/common/device-scan.c b/common/device-scan.c index 45b2530bb2..a34d86652f 100644 --- a/common/device-scan.c +++ b/common/device-scan.c @@ -51,6 +51,7 @@ #include "common/utils.h" #include "common/defs.h" #include "common/open-utils.h" +#include "common/string-utils.h" #include "common/units.h" static int btrfs_scan_done = 0; @@ -237,7 +238,7 @@ int btrfs_register_one_device(const char *fname) return -errno; } memset(&args, 0, sizeof(args)); - __strncpy_null(args.name, fname, sizeof(args.name)); + strncpy_null(args.name, fname, sizeof(args.name)); ret = ioctl(fd, BTRFS_IOC_SCAN_DEV, &args); if (ret < 0) { error("device scan failed on '%s': %m", fname); @@ -468,7 +469,7 @@ int btrfs_scan_devices(int verbose) if (!dev) continue; /* if we are here its definitely a btrfs disk*/ - __strncpy_null(path, blkid_dev_devname(dev), sizeof(path)); + strncpy_null(path, blkid_dev_devname(dev), sizeof(path)); if (stat(path, &dev_stat) < 0) continue; diff --git a/common/filesystem-utils.c b/common/filesystem-utils.c index 8dee27aab3..7e898a3d19 100644 --- a/common/filesystem-utils.c +++ b/common/filesystem-utils.c @@ -28,6 +28,7 @@ #include "common/filesystem-utils.h" #include "common/messages.h" #include "common/open-utils.h" +#include "common/string-utils.h" #include "common/path-utils.h" /* @@ -102,7 +103,7 @@ static int set_label_unmounted(const char *dev, const char *label) error_msg(ERROR_MSG_START_TRANS, "set label"); return PTR_ERR(trans); } - __strncpy_null(root->fs_info->super_copy->label, label, BTRFS_LABEL_SIZE); + strncpy_null(root->fs_info->super_copy->label, label, BTRFS_LABEL_SIZE); btrfs_commit_transaction(trans, root); @@ -123,7 +124,7 @@ static int set_label_mounted(const char *mount_path, const char *labelp) } memset(label, 0, sizeof(label)); - __strncpy_null(label, labelp, BTRFS_LABEL_SIZE); + strncpy_null(label, labelp, BTRFS_LABEL_SIZE); if (ioctl(fd, BTRFS_IOC_SET_FSLABEL, label) < 0) { error("unable to set label of %s: %m", mount_path); close(fd); @@ -152,8 +153,7 @@ int get_label_unmounted(const char *dev, char *label) if(!root) return -1; - __strncpy_null(label, root->fs_info->super_copy->label, - BTRFS_LABEL_SIZE); + strncpy_null(label, root->fs_info->super_copy->label, BTRFS_LABEL_SIZE); /* Now we close it since we are done. */ close_ctree(root); @@ -187,7 +187,7 @@ int get_label_mounted(const char *mount_path, char *labelp) return ret; } - __strncpy_null(labelp, label, BTRFS_LABEL_SIZE); + strncpy_null(labelp, label, BTRFS_LABEL_SIZE); close(fd); return 0; } diff --git a/common/path-utils.c b/common/path-utils.c index 9f907c40fa..04861d1668 100644 --- a/common/path-utils.c +++ b/common/path-utils.c @@ -35,6 +35,7 @@ */ #include #include +#include "common/string-utils.h" #include "common/path-utils.h" /* @@ -193,10 +194,10 @@ static int is_same_blk_file(const char* a, const char* b) char real_b[PATH_MAX]; if (!realpath(a, real_a)) - __strncpy_null(real_a, a, sizeof(real_a)); + strncpy_null(real_a, a, sizeof(real_a)); if (!realpath(b, real_b)) - __strncpy_null(real_b, b, sizeof(real_b)); + strncpy_null(real_b, b, sizeof(real_b)); /* Identical path? */ if (strcmp(real_a, real_b) == 0) @@ -345,26 +346,6 @@ char *path_canonicalize(const char *path) return canonical; } -/* - * __strncpy_null - strncpy with null termination - * @dest: the target array - * @src: the source string - * @n: maximum bytes to copy (size of *dest) - * - * Like strncpy, but ensures destination is null-terminated. - * - * Copies the string pointed to by src, including the terminating null - * byte ('\0'), to the buffer pointed to by dest, up to a maximum - * of n bytes. Then ensure that dest is null-terminated. - */ -char *__strncpy_null(char *dest, const char *src, size_t n) -{ - strncpy(dest, src, n); - if (n > 0) - dest[n - 1] = '\0'; - return dest; -} - /* * Test if path is a directory * Returns: @@ -403,7 +384,7 @@ int path_is_in_dir(const char *parent, const char *path) char *curr_dir = tmp; int ret; - __strncpy_null(tmp, path, sizeof(tmp)); + strncpy_null(tmp, path, sizeof(tmp)); while (strcmp(parent, curr_dir) != 0) { if (strcmp(curr_dir, "/") == 0) { @@ -432,7 +413,7 @@ int arg_copy_path(char *dest, const char *src, int destlen) if (len >= PATH_MAX || len >= destlen) return -ENAMETOOLONG; - __strncpy_null(dest, src, destlen); + strncpy_null(dest, src, destlen); return 0; } diff --git a/common/path-utils.h b/common/path-utils.h index da7e81e6db..558fa21adf 100644 --- a/common/path-utils.h +++ b/common/path-utils.h @@ -26,11 +26,6 @@ char *path_canonicalize(const char *path); int arg_copy_path(char *dest, const char *src, int destlen); int path_cat_out(char *out, const char *p1, const char *p2); int path_cat3_out(char *out, const char *p1, const char *p2, const char *p3); - -char *__strncpy_null(char *dest, const char *src, size_t n); -/* Helper to always get proper size of the destination string */ -#define strncpy_null(dest, src) __strncpy_null(dest, src, sizeof(dest)) - int path_is_block_device(const char *file); int path_is_a_mount_point(const char *file); int path_exists(const char *file); diff --git a/common/string-utils.c b/common/string-utils.c index c6e16ddcc4..d03f51c19c 100644 --- a/common/string-utils.c +++ b/common/string-utils.c @@ -44,6 +44,26 @@ int string_has_prefix(const char *str, const char *prefix) return (unsigned char)*prefix - (unsigned char)*str; } +/* + * strncpy_null - strncpy with null termination + * @dest: the target array + * @src: the source string + * @n: maximum bytes to copy (size of *dest) + * + * Like strncpy, but ensures destination is null-terminated. + * + * Copies the string pointed to by src, including the terminating null + * byte ('\0'), to the buffer pointed to by dest, up to a maximum + * of n bytes. Then ensure that dest is null-terminated. + */ +char *strncpy_null(char *dest, const char *src, size_t n) +{ + strncpy(dest, src, n); + if (n > 0) + dest[n - 1] = '\0'; + return dest; +} + /* * This function should be only used when parsing command arg, it won't return * error to its caller and rather exit directly just like usage(). diff --git a/common/string-utils.h b/common/string-utils.h index 1a46315cad..7b4225e4cb 100644 --- a/common/string-utils.h +++ b/common/string-utils.h @@ -22,6 +22,8 @@ int string_is_numerical(const char *str); int string_has_prefix(const char *str, const char *prefix); +char *strncpy_null(char *dest, const char *src, size_t n); + /* * Helpers prefixed by arg_* can exit if the argument is invalid and are supposed * to be used when parsing command line options where the immediate exit is valid diff --git a/convert/common.c b/convert/common.c index d0d3e087c7..b093fdb5f1 100644 --- a/convert/common.c +++ b/convert/common.c @@ -30,6 +30,7 @@ #include "kernel-shared/uapi/btrfs_tree.h" #include "common/path-utils.h" #include "common/messages.h" +#include "common/string-utils.h" #include "common/fsfeatures.h" #include "mkfs/common.h" #include "convert/common.h" @@ -148,7 +149,7 @@ static int setup_temp_super(int fd, struct btrfs_mkfs_config *cfg, btrfs_set_super_cache_generation(&super, -1); btrfs_set_super_incompat_flags(&super, cfg->features.incompat_flags); if (cfg->label) - __strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE); + strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE); /* Sys chunk array will be re-initialized at chunk tree init time */ super.sys_chunk_array_size = 0; diff --git a/convert/main.c b/convert/main.c index 3d2fd61a71..48bcff2fa3 100644 --- a/convert/main.c +++ b/convert/main.c @@ -1322,8 +1322,7 @@ static int do_convert(const char *devname, u32 convert_flags, u32 nodesize, memset(root->fs_info->super_copy->label, 0, BTRFS_LABEL_SIZE); if (convert_flags & CONVERT_FLAG_COPY_LABEL) { - __strncpy_null(root->fs_info->super_copy->label, - cctx.label, BTRFS_LABEL_SIZE); + strncpy_null(root->fs_info->super_copy->label, cctx.label, BTRFS_LABEL_SIZE); printf("Copy label '%s'\n", root->fs_info->super_copy->label); } else if (convert_flags & CONVERT_FLAG_SET_LABEL) { strcpy(root->fs_info->super_copy->label, fslabel); @@ -1938,7 +1937,7 @@ int BOX_MAIN(convert)(int argc, char *argv[]) "label too long, trimmed to %d bytes", BTRFS_LABEL_SIZE - 1); } - __strncpy_null(fslabel, optarg, BTRFS_LABEL_SIZE); + strncpy_null(fslabel, optarg, BTRFS_LABEL_SIZE); break; case 'L': copylabel = CONVERT_FLAG_COPY_LABEL; diff --git a/mkfs/common.c b/mkfs/common.c index ee0284bfd4..5de3e0b6d5 100644 --- a/mkfs/common.c +++ b/mkfs/common.c @@ -38,6 +38,7 @@ #include "common/path-utils.h" #include "common/device-utils.h" #include "common/open-utils.h" +#include "common/string-utils.h" #include "mkfs/common.h" static u64 reference_root_table[] = { @@ -476,7 +477,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg) btrfs_set_super_nr_global_roots(&super, 1); if (cfg->label) - __strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE); + strncpy_null(super.label, cfg->label, BTRFS_LABEL_SIZE); /* create the tree of root objects */ memset(buf->data, 0, cfg->nodesize); From a35409328e228f43f39136296908880be699053f Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 20:56:38 +0200 Subject: [PATCH 21/35] btrfs-progs: use strncpy_null everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the safe version of strncpy that makes sure the string is terminated. To be noted: - the conversion in scrub path handling was skipped - sizes of device paths in some ioctl related structures is BTRFS_DEVICE_PATH_NAME_MAX + 1 Recently gcc 13.3 started to detect problems with our use of strncpy potentially lacking the null terminator, warnings like: cmds/inspect.c: In function ‘cmd_inspect_logical_resolve’: cmds/inspect.c:294:33: warning: ‘__builtin_strncpy’ specified bound 4096 equals destination size [-Wstringop-truncation] 294 | strncpy(mount_path, mounted, PATH_MAX); | ^ Signed-off-by: David Sterba --- btrfs-corrupt-block.c | 2 +- cmds/device.c | 5 ++--- cmds/inspect.c | 4 ++-- cmds/replace.c | 8 ++++---- cmds/restore.c | 3 +-- cmds/scrub.c | 2 +- cmds/subvolume-list.c | 6 ++---- common/help.c | 3 +-- common/open-utils.c | 8 ++++---- convert/main.c | 2 +- convert/source-ext2.c | 3 ++- kernel-shared/print-tree.c | 9 +++++---- mkfs/main.c | 5 ++--- 13 files changed, 28 insertions(+), 32 deletions(-) diff --git a/btrfs-corrupt-block.c b/btrfs-corrupt-block.c index e883198919..fc71eb2cbf 100644 --- a/btrfs-corrupt-block.c +++ b/btrfs-corrupt-block.c @@ -1398,7 +1398,7 @@ int main(int argc, char **argv) inode = arg_strtou64(optarg); break; case 'f': - strncpy(field, optarg, FIELD_BUF_LEN); + strncpy_null(field, optarg, FIELD_BUF_LEN); break; case 'x': file_extent = arg_strtou64(optarg); diff --git a/cmds/device.c b/cmds/device.c index 6a9a4db202..2a08dd5c32 100644 --- a/cmds/device.c +++ b/cmds/device.c @@ -799,9 +799,8 @@ static int cmd_device_stats(const struct cmd_struct *cmd, int argc, char **argv) char path[BTRFS_DEVICE_PATH_NAME_MAX + 1]; int err2; - strncpy(path, (char *)di_args[i].path, - BTRFS_DEVICE_PATH_NAME_MAX); - path[BTRFS_DEVICE_PATH_NAME_MAX] = 0; + strncpy_null(path, (char *)di_args[i].path, + BTRFS_DEVICE_PATH_NAME_MAX + 1); args.devid = di_args[i].devid; args.nr_items = BTRFS_DEV_STAT_VALUES_MAX; diff --git a/cmds/inspect.c b/cmds/inspect.c index 5b3f3c0534..353ed639ee 100644 --- a/cmds/inspect.c +++ b/cmds/inspect.c @@ -258,7 +258,7 @@ static int cmd_inspect_logical_resolve(const struct cmd_struct *cmd, if (name[0] == 0) { path_ptr[-1] = '\0'; path_fd = fd; - strncpy(mount_path, full_path, PATH_MAX); + strncpy_null(mount_path, full_path, PATH_MAX); } else { char *mounted = NULL; char subvol[PATH_MAX]; @@ -291,7 +291,7 @@ static int cmd_inspect_logical_resolve(const struct cmd_struct *cmd, continue; } - strncpy(mount_path, mounted, PATH_MAX); + strncpy_null(mount_path, mounted, PATH_MAX); free(mounted); path_fd = btrfs_open_dir(mount_path); diff --git a/cmds/replace.c b/cmds/replace.c index c1eec84c79..5f1222b241 100644 --- a/cmds/replace.c +++ b/cmds/replace.c @@ -266,8 +266,8 @@ static int cmd_replace_start(const struct cmd_struct *cmd, goto leave_with_error; } } else if (path_is_block_device(srcdev) > 0) { - strncpy((char *)start_args.start.srcdev_name, srcdev, - BTRFS_DEVICE_PATH_NAME_MAX); + strncpy_null((char *)start_args.start.srcdev_name, srcdev, + BTRFS_DEVICE_PATH_NAME_MAX + 1); start_args.start.srcdevid = 0; srcdev_size = device_get_partition_size(srcdev); } else { @@ -292,8 +292,8 @@ static int cmd_replace_start(const struct cmd_struct *cmd, goto leave_with_error; } - strncpy((char *)start_args.start.tgtdev_name, dstdev, - BTRFS_DEVICE_PATH_NAME_MAX); + strncpy_null((char *)start_args.start.tgtdev_name, dstdev, + BTRFS_DEVICE_PATH_NAME_MAX + 1); ret = btrfs_prepare_device(fddstdev, dstdev, &dstdev_block_count, 0, PREP_DEVICE_ZERO_END | PREP_DEVICE_VERBOSE | (discard ? PREP_DEVICE_DISCARD : 0) | diff --git a/cmds/restore.c b/cmds/restore.c index ca2a13a1fe..6bc619b347 100644 --- a/cmds/restore.c +++ b/cmds/restore.c @@ -1535,8 +1535,7 @@ static int cmd_restore(const struct cmd_struct *cmd, int argc, char **argv) ret = 1; goto out; } - strncpy(dir_name, argv[optind + 1], sizeof dir_name); - dir_name[sizeof dir_name - 1] = 0; + strncpy_null(dir_name, argv[optind + 1], sizeof(dir_name)); /* Strip the trailing / on the dir name */ len = strlen(dir_name); diff --git a/cmds/scrub.c b/cmds/scrub.c index 03c1f7b1d6..202690dbe9 100644 --- a/cmds/scrub.c +++ b/cmds/scrub.c @@ -1437,7 +1437,7 @@ static int scrub_start(const struct cmd_struct *cmd, int argc, char **argv, sock_path, sizeof(sock_path)); /* ignore EOVERFLOW, try using a shorter path for the socket */ addr.sun_path[sizeof(addr.sun_path) - 1] = '\0'; - strncpy(addr.sun_path, sock_path, sizeof(addr.sun_path) - 1); + strncpy_null(addr.sun_path, sock_path, sizeof(addr.sun_path)); ret = bind(prg_fd, (struct sockaddr *)&addr, sizeof(addr)); if (ret != -1 || errno != EADDRINUSE) break; diff --git a/cmds/subvolume-list.c b/cmds/subvolume-list.c index 21319267e2..52a30aa00f 100644 --- a/cmds/subvolume-list.c +++ b/cmds/subvolume-list.c @@ -547,8 +547,7 @@ static int update_root(struct rb_root *root_lookup, error_msg(ERROR_MSG_MEMORY, NULL); exit(1); } - strncpy(ri->name, name, name_len); - ri->name[name_len] = 0; + strncpy_null(ri->name, name, name_len); } if (ref_tree) ri->ref_tree = ref_tree; @@ -619,8 +618,7 @@ static int add_root(struct rb_root *root_lookup, error_msg(ERROR_MSG_MEMORY, NULL); exit(1); } - strncpy(ri->name, name, name_len); - ri->name[name_len] = 0; + strncpy_null(ri->name, name, name_len); } if (ref_tree) ri->ref_tree = ref_tree; diff --git a/common/help.c b/common/help.c index c302d375be..6cf8e2a9b2 100644 --- a/common/help.c +++ b/common/help.c @@ -47,8 +47,7 @@ void fixup_argv0(char **argv, const char *token) void set_argv0(char **argv) { - strncpy(argv0_buf, argv[0], sizeof(argv0_buf)); - argv0_buf[sizeof(argv0_buf) - 1] = 0; + strncpy_null(argv0_buf, argv[0], sizeof(argv0_buf)); } int check_argc_exact(int nargs, int expected) diff --git a/common/open-utils.c b/common/open-utils.c index 95294f63e9..8490be4af0 100644 --- a/common/open-utils.c +++ b/common/open-utils.c @@ -32,6 +32,7 @@ #include "common/messages.h" #include "common/path-utils.h" #include "common/device-scan.h" +#include "common/string-utils.h" #include "common/open-utils.h" /* @@ -103,10 +104,9 @@ int check_mounted_where(int fd, const char *file, char *where, int size, } /* Did we find an entry in mnt table? */ - if (mnt && size && where) { - strncpy(where, mnt->mnt_dir, size); - where[size-1] = 0; - } + if (mnt && size && where) + strncpy_null(where, mnt->mnt_dir, size); + if (fs_dev_ret) *fs_dev_ret = fs_devices_mnt; else if (noscan) diff --git a/convert/main.c b/convert/main.c index 48bcff2fa3..8e73aa25b1 100644 --- a/convert/main.c +++ b/convert/main.c @@ -1996,7 +1996,7 @@ int BOX_MAIN(convert)(int argc, char *argv[]) error("invalid UUID: %s\n", optarg); return 1; } - strncpy(fsid, optarg, sizeof(fsid)); + strncpy_null(fsid, optarg, sizeof(fsid)); } break; case GETOPT_VAL_HELP: diff --git a/convert/source-ext2.c b/convert/source-ext2.c index 00a7b290cb..5efaaa4b0b 100644 --- a/convert/source-ext2.c +++ b/convert/source-ext2.c @@ -30,6 +30,7 @@ #include "kernel-shared/file-item.h" #include "common/extent-cache.h" #include "common/messages.h" +#include "common/string-utils.h" #include "convert/common.h" #include "convert/source-fs.h" #include "convert/source-ext2.h" @@ -638,7 +639,7 @@ static int ext2_copy_single_xattr(struct btrfs_trans_handle *trans, data = databuf; datalen = bufsize; } - strncpy(namebuf, xattr_prefix_table[name_index], XATTR_NAME_MAX); + strncpy_null(namebuf, xattr_prefix_table[name_index], XATTR_NAME_MAX); strncat(namebuf, EXT2_EXT_ATTR_NAME(entry), entry->e_name_len); if (name_len + datalen > BTRFS_LEAF_DATA_SIZE(root->fs_info) - sizeof(struct btrfs_item) - sizeof(struct btrfs_dir_item)) { diff --git a/kernel-shared/print-tree.c b/kernel-shared/print-tree.c index 568a84cb35..6f78ec3512 100644 --- a/kernel-shared/print-tree.c +++ b/kernel-shared/print-tree.c @@ -36,6 +36,7 @@ #include "common/defs.h" #include "common/internal.h" #include "common/messages.h" +#include "common/string-utils.h" #include "uapi/btrfs.h" static void print_dir_item_type(struct extent_buffer *eb, @@ -186,7 +187,7 @@ static void bg_flags_to_str(u64 flags, char *ret) ret[0] = '\0'; if (flags & BTRFS_BLOCK_GROUP_DATA) { empty = 0; - strncpy(ret, "DATA", BG_FLAG_STRING_LEN); + strncpy_null(ret, "DATA", BG_FLAG_STRING_LEN); } if (flags & BTRFS_BLOCK_GROUP_METADATA) { if (!empty) @@ -209,7 +210,7 @@ static void bg_flags_to_str(u64 flags, char *ret) * Thus here we only fill @profile if it's not single. */ if (strncmp(name, "SINGLE", strlen("SINGLE")) != 0) - strncpy(profile, name, BG_FLAG_STRING_LEN); + strncpy_null(profile, name, BG_FLAG_STRING_LEN); } if (profile[0]) { strncat(ret, "|", BG_FLAG_STRING_LEN); @@ -1398,7 +1399,7 @@ static void print_header_info(struct extent_buffer *eb, unsigned int mode) #define DEV_REPLACE_STRING_LEN 64 #define CASE_DEV_REPLACE_MODE_ENTRY(dest, name) \ case BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_##name: \ - strncpy((dest), #name, DEV_REPLACE_STRING_LEN); \ + strncpy_null((dest), #name, DEV_REPLACE_STRING_LEN); \ break; static void replace_mode_to_str(u64 flags, char *ret) @@ -1415,7 +1416,7 @@ static void replace_mode_to_str(u64 flags, char *ret) #define CASE_DEV_REPLACE_STATE_ENTRY(dest, name) \ case BTRFS_IOCTL_DEV_REPLACE_STATE_##name: \ - strncpy((dest), #name, DEV_REPLACE_STRING_LEN); \ + strncpy_null((dest), #name, DEV_REPLACE_STRING_LEN); \ break; static void replace_state_to_str(u64 flags, char *ret) diff --git a/mkfs/main.c b/mkfs/main.c index 8d0924ea3b..b40f7432bb 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1361,8 +1361,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv) source_dir = strdup(optarg); break; case 'U': - strncpy(fs_uuid, optarg, - BTRFS_UUID_UNPARSED_SIZE - 1); + strncpy_null(fs_uuid, optarg, BTRFS_UUID_UNPARSED_SIZE); break; case 'K': opt_discard = false; @@ -1371,7 +1370,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv) bconf_be_quiet(); break; case GETOPT_VAL_DEVICE_UUID: - strncpy(dev_uuid, optarg, BTRFS_UUID_UNPARSED_SIZE - 1); + strncpy_null(dev_uuid, optarg, BTRFS_UUID_UNPARSED_SIZE); break; case GETOPT_VAL_SHRINK: shrink_rootdir = true; From 7420ee772152f8c8eeb811b53d9dc589fa89de15 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 22:25:28 +0200 Subject: [PATCH 22/35] btrfs-progs: receive dump: encode clone path and xattr name/value The xattr names are user strings but still can potentially contain special characters (as reported). There doesn't seem to be a restriction on the name defined. The xattr values care length-encoded byte arrays so escaping needs be done. The clone source is a path and by mistake lacked the encoding. Issue: #818 Signed-off-by: David Sterba --- cmds/receive-dump.c | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c index 83d22a8784..c3e711be12 100644 --- a/cmds/receive-dump.c +++ b/cmds/receive-dump.c @@ -45,10 +45,9 @@ * Returns the length of the escaped characters. Unprintable characters are * escaped as octals. */ -static int print_path_escaped(const char *path) +static int print_path_escaped_len(const char *path, size_t path_len) { size_t i; - size_t path_len = strlen(path); int len = 0; for (i = 0; i < path_len; i++) { @@ -81,6 +80,11 @@ static int print_path_escaped(const char *path) return len; } +static int print_path_escaped(const char *path) +{ + return print_path_escaped_len(path, strlen(path)); +} + enum print_mode { PRINT_DUMP_NORMAL, PRINT_DUMP_SUBVOLUME, @@ -251,24 +255,37 @@ static int print_clone(const char *path, u64 offset, u64 len, char full_path[PATH_MAX]; int ret; - PATH_CAT_OR_RET("clone", full_path, r->full_subvol_path, clone_path, - ret); - return PRINT_DUMP(user, path, "clone", - "offset=%llu len=%llu from=%s clone_offset=%llu", - offset, len, full_path, clone_offset); + PATH_CAT_OR_RET("clone", full_path, r->full_subvol_path, clone_path, ret); + PRINT_DUMP_NO_NEWLINE(user, path, "clone", "offset=%llu len=%llu from=", offset, len); + print_path_escaped(full_path); + putchar(' '); + printf("clone_offset=%llu\n", clone_offset); + return 0; } +/* + * Xattr names are like paths and can potentially contain special characters, + * xattr values can be arbitrary. + */ static int print_set_xattr(const char *path, const char *name, const void *data, int len, void *user) { - return PRINT_DUMP(user, path, "set_xattr", "name=%s data=%.*s len=%d", - name, len, (char *)data, len); + PRINT_DUMP_NO_NEWLINE(user, path, "set_xattr", "name="); + print_path_escaped(name); + putchar(' '); + print_path_escaped_len((char *)data, len); + putchar(' '); + printf("len=%d\n", len); + return 0; } static int print_remove_xattr(const char *path, const char *name, void *user) { - return PRINT_DUMP(user, path, "remove_xattr", "name=%s", name); + PRINT_DUMP_NO_NEWLINE(user, path, "remove_xattr", "name="); + print_path_escaped(name); + putchar('\n'); + return 0; } static int print_truncate(const char *path, u64 size, void *user) From 751c57f95024760093f126955e30187c58eae74e Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 22:38:33 +0200 Subject: [PATCH 23/35] btrfs-progs: tests: update misc/054 to test escaping of clone and xattrs Add cases for clone source path and xattr name and value escapin in receive dump. Signed-off-by: David Sterba --- .../054-receive-dump-newlines/test.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/misc-tests/054-receive-dump-newlines/test.sh b/tests/misc-tests/054-receive-dump-newlines/test.sh index c76feed6d6..cab1b52343 100755 --- a/tests/misc-tests/054-receive-dump-newlines/test.sh +++ b/tests/misc-tests/054-receive-dump-newlines/test.sh @@ -1,6 +1,6 @@ #!/bin/bash # Verify that receive --dump escapes paths for rename, symlink and hardlink -# when it's the "dest=" value +# when it's the "dest=" value, clone (source path) and xattr name and value source "$TEST_TOP/common" || exit @@ -19,7 +19,7 @@ run_check $SUDO_HELPER ln -s "$TEST_MNT/subv1/file failed symlink source" "$TEST_MNT/subv1/file failed symlink target" -# Hardlinke +# Hardlink run_check $SUDO_HELPER touch "$TEST_MNT/subv1/file failed link source" @@ -27,6 +27,19 @@ run_check $SUDO_HELPER ln "$TEST_MNT/subv1/file failed link source" "$TEST_MNT/subv1/file failed link target" +# Xattr name and value, create +run_check $SUDO_HELPER touch "$TEST_MNT/subv1/xattr-file" +run_check $SUDO_HELPER setfattr -n user.'name +failed xattr name' -v '123 +failed xattr value' "$TEST_MNT/subv1/xattr-file" + +# Clone +run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/subv1/clone-source +failed clone source write" bs=1M count=1 +run_check $SUDO_HELPER cp --reflink=always "$TEST_MNT/subv1/clone-source +failed clone source write" "$TEST_MNT/subv1/clone-target +failed clone target" + run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$TEST_MNT/subv1" "$TEST_MNT/snap1" _mktemp_local send.stream run_check $SUDO_HELPER "$TOP/btrfs" send -f send.stream "$TEST_MNT/snap1" From 8b240d0625e11a4061f0935cefbfc3761ac85cf5 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 22:43:53 +0200 Subject: [PATCH 24/35] btrfs-progs: receive dump: fix formatting of encoded write message The separator of key=value is only one or more space character, the 'encoded_write' also uses ',' which is inconsistent with the rest. Signed-off-by: David Sterba --- cmds/receive-dump.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c index c3e711be12..503da32920 100644 --- a/cmds/receive-dump.c +++ b/cmds/receive-dump.c @@ -353,9 +353,7 @@ static int print_encoded_write(const char *path, const void *data, u64 offset, u32 compression, u32 encryption, void *user) { return PRINT_DUMP(user, path, "encoded_write", - "offset=%llu len=%llu, unencoded_file_len=%llu, " - "unencoded_len=%llu, unencoded_offset=%llu, " - "compression=%u, encryption=%u", +"offset=%llu len=%llu unencoded_file_len=%llu unencoded_len=%llu unencoded_offset=%llu compression=%u encryption=%u", offset, len, unencoded_file_len, unencoded_len, unencoded_offset, compression, encryption); } From 1487e61a2f96dc1b915805b23fcb3a615837fa81 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 22:53:42 +0200 Subject: [PATCH 25/35] btrfs-progs: docs: clarify receive --dump encoding [ci skip] Signed-off-by: David Sterba --- Documentation/btrfs-receive.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/btrfs-receive.rst b/Documentation/btrfs-receive.rst index f0ae73d78f..85939cd580 100644 --- a/Documentation/btrfs-receive.rst +++ b/Documentation/btrfs-receive.rst @@ -66,6 +66,11 @@ A subvolume is made read-only after the receiving process finishes successfully dump the stream metadata, one line per operation Does not require the *path* parameter. The filesystem remains unchanged. + Each stream command is on one line in the form of *key=value* and separated + by one or more spaces. Values that contain special characters (like + paths or extended attributes) are encoded in C-like way, e.g. *'\\n'* or + octal escape sequence like *'\\NNN'* where N is the char value. Same encoding + as is used in */proc* files. -q|--quiet (deprecated) alias for global *-q* option From 7535c35128958dc0471fe958da8b31fd7e126de0 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Jun 2024 18:09:57 +0200 Subject: [PATCH 26/35] btrfs-progs: fi df: extended information about profile types There's more information available in sysfs (/sys/fs/btrfs/FSID/allocation) that we can print in 'fi df'. This is still meant for debugging or deeper analysis of the filesystem, the values need to be correctly interpreted with respect to the profiles, persistence and other conditonal features. The extended output is not printed by default and for now is behind the verbosity options: $ btrfs -vv fi df /mnt Data, single: total=47.06GiB, used=25.32GiB System, DUP: total=32.00MiB, used=16.00KiB Metadata, DUP: total=1.44GiB, used=961.20MiB GlobalReserve, single: total=125.62MiB, used=0.00B Data: bg_reclaim_threshold 0% bytes_may_use 8.00KiB bytes_pinned 0.00B bytes_readonly 64.00KiB bytes_reserved 0.00B bytes_used 25.32GiB bytes_zone_unusable 0.00B chunk_size 10.00GiB disk_total 47.06GiB disk_used 25.32GiB total_bytes 47.06GiB Metadata: bg_reclaim_threshold 0% bytes_may_use 126.62MiB bytes_pinned 0.00B bytes_readonly 0.00B bytes_reserved 0.00B bytes_used 961.20MiB bytes_zone_unusable 0.00B chunk_size 256.00MiB disk_total 2.88GiB disk_used 1.88GiB total_bytes 1.44GiB System: bg_reclaim_threshold 0% bytes_may_use 0.00B bytes_pinned 0.00B bytes_readonly 0.00B bytes_reserved 0.00B bytes_used 16.00KiB bytes_zone_unusable 0.00B chunk_size 32.00MiB disk_total 64.00MiB disk_used 32.00KiB total_bytes 32.00MiB Signed-off-by: David Sterba --- cmds/filesystem.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cmds/filesystem.c b/cmds/filesystem.c index 291abeedf3..6c1c16b998 100644 --- a/cmds/filesystem.c +++ b/cmds/filesystem.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include "libbtrfsutil/btrfsutil.h" #include "kernel-lib/list.h" @@ -53,6 +54,7 @@ #include "common/device-utils.h" #include "common/open-utils.h" #include "common/parse-utils.h" +#include "common/sysfs-utils.h" #include "common/string-utils.h" #include "common/filesystem-utils.h" #include "common/format-output.h" @@ -81,6 +83,41 @@ static const char * const cmd_filesystem_df_usage[] = { NULL }; +static void print_df_by_type(int fd, unsigned int unit_mode) { + static const char *files[] = { + "bg_reclaim_threshold", + "bytes_may_use", + "bytes_pinned", + "bytes_readonly", + "bytes_reserved", + "bytes_used", + "bytes_zone_unusable", + "chunk_size", + "disk_total", + "disk_used", + "total_bytes", + }; + char path[PATH_MAX] = { 0 }; + const char *types[] = { "data", "metadata", "mixed", "system" }; + u64 tmp; + int ret; + + for (int ti = 0; ti < ARRAY_SIZE(types); ti++) { + for (int i = 0; i < ARRAY_SIZE(files); i++) { + path_cat3_out(path, "allocation", types[ti], files[i]); + ret = sysfs_read_fsid_file_u64(fd, path, &tmp); + if (ret < 0) + continue; + if (i == 0) + pr_verbose(LOG_INFO, "%c%s:\n", toupper(types[ti][0]), types[ti] + 1); + if (strcmp(files[i], "bg_reclaim_threshold") == 0) + pr_verbose(LOG_INFO, " %-24s %14llu%%\n", files[i], tmp); + else + pr_verbose(LOG_INFO, " %-24s %16s\n", files[i], pretty_size_mode(tmp, unit_mode)); + } + } +} + static void print_df_text(int fd, struct btrfs_ioctl_space_args *sargs, unsigned unit_mode) { u64 i; @@ -100,6 +137,7 @@ static void print_df_text(int fd, struct btrfs_ioctl_space_args *sargs, unsigned (ok ? ", zone_unusable=" : ""), (ok ? pretty_size_mode(unusable, unit_mode) : "")); } + print_df_by_type(fd, unit_mode); } static const struct rowspec filesystem_df_rowspec[] = { From 410d82d7b11676547f63671243e3dbfba1681ada Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 18 Jun 2024 00:11:11 +0200 Subject: [PATCH 27/35] btrfs-progs: tests: fix shellcheck reports in cli-tests - variable quoting - cd error handling - `` to $() - command output instead of command (008-subvolume-get-set-default) Signed-off-by: David Sterba --- tests/cli-tests/001-btrfs/test.sh | 2 +- tests/cli-tests/004-send-parent-multi-subvol/test.sh | 2 +- tests/cli-tests/008-subvolume-get-set-default/test.sh | 4 ++-- tests/cli-tests/014-multiple-profiles-warning/test.sh | 8 ++++---- tests/cli-tests/016-btrfs-fi-usage/test.sh | 6 +++--- tests/cli-tests/024-scrub-limit/test.sh | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/cli-tests/001-btrfs/test.sh b/tests/cli-tests/001-btrfs/test.sh index 0f0d6715fd..85213fccf0 100755 --- a/tests/cli-tests/001-btrfs/test.sh +++ b/tests/cli-tests/001-btrfs/test.sh @@ -6,7 +6,7 @@ source "$TEST_TOP/common" || exit check_prereq btrfs # returns 1 -run_mayfail $TOP/btrfs || true +run_mayfail "$TOP/btrfs" || true run_check "$TOP/btrfs" version run_check "$TOP/btrfs" version -- run_check "$TOP/btrfs" help diff --git a/tests/cli-tests/004-send-parent-multi-subvol/test.sh b/tests/cli-tests/004-send-parent-multi-subvol/test.sh index 94efcb8622..cabc2a2d69 100755 --- a/tests/cli-tests/004-send-parent-multi-subvol/test.sh +++ b/tests/cli-tests/004-send-parent-multi-subvol/test.sh @@ -13,7 +13,7 @@ prepare_test_dev run_check_mkfs_test_dev run_check_mount_test_dev -here=`pwd` +here=$(pwd) cd "$TEST_MNT" || _fail "cannot chdir to TEST_MNT" run_check $SUDO_HELPER "$TOP/btrfs" subvolume create subv-parent diff --git a/tests/cli-tests/008-subvolume-get-set-default/test.sh b/tests/cli-tests/008-subvolume-get-set-default/test.sh index 4e536ed7a2..2814fba344 100755 --- a/tests/cli-tests/008-subvolume-get-set-default/test.sh +++ b/tests/cli-tests/008-subvolume-get-set-default/test.sh @@ -13,14 +13,14 @@ check_default_id() { id=$(run_check_stdout $SUDO_HELPER "$TOP/btrfs" subvolume get-default .) \ || { echo "$id"; exit 1; } - if $(echo "$id" | grep -vq "ID $1"); then + if echo "$id" | grep -vq "ID $1"; then _fail "subvolume get-default: default id is not $1, but $id" fi } run_check_mkfs_test_dev run_check_mount_test_dev -cd "$TEST_MNT" +cd "$TEST_MNT" || _fail "Cannot cd into TEST_MNT $TEST_MNT" check_default_id 5 diff --git a/tests/cli-tests/014-multiple-profiles-warning/test.sh b/tests/cli-tests/014-multiple-profiles-warning/test.sh index 1e374e4112..8fbbc9cb1e 100755 --- a/tests/cli-tests/014-multiple-profiles-warning/test.sh +++ b/tests/cli-tests/014-multiple-profiles-warning/test.sh @@ -62,8 +62,8 @@ test_run_commands() { run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d single -m single "${loopdevs[@]}" run_check_mount_test_dev run_check "$TOP/btrfs" filesystem usage "$TEST_MNT" -for i in `seq 10`; do - run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file$i bs=100M count=1 status=none +for i in $(seq 10); do + run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file$i" bs=100M count=1 status=none done # Create filesystem with single and RAID1 profiles run_check $SUDO_HELPER "$TOP/btrfs" balance start -dconvert=raid1,limit=1 "$TEST_MNT" @@ -76,8 +76,8 @@ run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f --mixed -d single -m single "${loopd run_check_mount_test_dev run_check "$TOP/btrfs" filesystem usage "$TEST_MNT" # Create 1 and a half of 1G chunks -for i in `seq 14`; do - run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT"/file$i bs=100M count=1 status=none +for i in $(seq 14); do + run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file$i" bs=100M count=1 status=none done # Create filesystem with single and RAID1 profiles, the limit=1 trick does not work # so use the usage filter to convert about half of the filesystem diff --git a/tests/cli-tests/016-btrfs-fi-usage/test.sh b/tests/cli-tests/016-btrfs-fi-usage/test.sh index ff1f97881a..20b716fbee 100755 --- a/tests/cli-tests/016-btrfs-fi-usage/test.sh +++ b/tests/cli-tests/016-btrfs-fi-usage/test.sh @@ -63,7 +63,7 @@ test_raid1() set -- $i IFS=$OLDIFS - run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d"$1" ${loopdevs[@]} + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d"$1" "${loopdevs[@]}" run_check_mount_test_dev vars=($(report_numbers)) data_chunk_size=${vars[1]} @@ -87,7 +87,7 @@ test_raid0() local used_on_dev local data_ratio - run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -draid0 ${loopdevs[@]} + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -draid0 "${loopdevs[@]}" run_check_mount_test_dev vars=($(report_numbers)) data_chunk_size=${vars[1]} @@ -118,7 +118,7 @@ test_raid56() set -- $i IFS=$OLDIFS - run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d"$1" ${loopdevs[@]} + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d"$1" "${loopdevs[@]}" run_check_mount_test_dev vars=($(report_numbers)) data_chunk_size=${vars[1]} diff --git a/tests/cli-tests/024-scrub-limit/test.sh b/tests/cli-tests/024-scrub-limit/test.sh index d672722d5b..000371f2fa 100755 --- a/tests/cli-tests/024-scrub-limit/test.sh +++ b/tests/cli-tests/024-scrub-limit/test.sh @@ -36,11 +36,11 @@ fi # Set the limits by command here=`pwd` -cd "$sysfs/devinfo" +cd "$sysfs/devinfo" || _fail "Cannot cd to $sysfs/devinfo" for i in *; do run_check $SUDO_HELPER "$TOP/btrfs" scrub limit -d "$i" -l 20m "$TEST_MNT" done -cd "$here" +cd "$here" || _fail "Cannot cd to $here" run_check "$TOP/btrfs" scrub limit "$TEST_MNT" # Set limits for all devices From 7c10c721342277da5edbc47594539362c72a5157 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 19 Jun 2024 02:29:45 +0200 Subject: [PATCH 28/35] btrfs-progs: update sorting API Add parsing of user defined sorting specification and some helpers. Example usage: sortdef = "key1,key2,key3" do { id = compare_parse_key_to_id(&comp, sortdef) if (id < 0) return error; compare_add_sort_id(&comp, id) } while(id >= 0); Signed-off-by: David Sterba --- common/sort-utils.c | 156 ++++++++++++++++++++++++++++++++++++++++++++ common/sort-utils.h | 17 ++++- 2 files changed, 171 insertions(+), 2 deletions(-) diff --git a/common/sort-utils.c b/common/sort-utils.c index d5b15a1a3a..bae2cd160b 100644 --- a/common/sort-utils.c +++ b/common/sort-utils.c @@ -16,7 +16,9 @@ #include #include +#include #include "common/sort-utils.h" +#include "common/messages.h" int compare_init(struct compare *comp, const struct sortdef *sortdef) { @@ -55,3 +57,157 @@ int compare_add_sort_key(struct compare *comp, const char *key) } return 0; } + +/* + * Append given sort by its @id from associated sortdef. + * + * Return: 0 if id is valid + * -1 if id not in sortdef + */ +int compare_add_sort_id(struct compare *comp, int id) +{ + int i; + + if (!comp->sortdef) + return -1; + + if (id < 0) + return -1; + + for (i = 0; i < SORT_MAX_KEYS; i++) { + if (comp->sortdef[i].name == NULL) + return -1; + if (comp->sortdef[i].id == id) { + comp->comp[comp->count] = comp->sortdef[i].comp; + comp->count++; + break; + } + } + return 0; +} + +/* + * Consume word-like list of key names (coma separated) and return its id if + * found in sortdef. The @next pointer is advanced to the next expected key start. + * Empty and NULL @next is accepted. + * + * Key lookup is case insensitive. + * + * Retrun: id from sortdef if a matching + * -1 on error + * -2 end of buffer + */ +int compare_parse_key_to_id(const struct compare *comp, const char **next) +{ + const char *tmp = *next, *start = *next; + + if (!comp->sortdef) + return -1; + + /* No sort string (use defaults), or last. */ + if (!*next || !**next) + return -2; + + do { + /* End of word. */ + if (*tmp == ',' || *tmp == 0) { + /* Look up in sortdef. */ + for (int i = 0; comp->sortdef[i].name; i++) { + int len = strlen(comp->sortdef[i].name); + + if (strncasecmp(start, comp->sortdef[i].name, len) == 0) { + /* Point to last NUL. */ + *next = tmp; + /* Or the next valid char. */ + if (*tmp) + (*next)++; + return comp->sortdef[i].id; + } + } + /* Not found, report which one. */ + *next = start; + return -1; + } + /* Invalid char found. */ + if (!isalnum(*tmp)) { + *next = tmp; + return -1; + } + tmp++; + } while(1); + + /* Not found. */ + *next = start; + return -1; +} + +/* Read id of its associated sort @key. Key lookup is case insensitive. */ +int compare_key_id(const struct compare *comp, const char *key) +{ + if (!comp->sortdef) + return -1; + + for (int i = 0; comp->sortdef[i].name; i++) + if (strcasecmp(comp->sortdef[i].name, key) == 0) + return comp->sortdef[i].id; + return -1; +} + +/* Read sort key name associated to @id. */ +const char *compare_id_name(const struct compare *comp, int id) +{ + if (!comp->sortdef) + return NULL; + + for (int i = 0; comp->sortdef[i].name; i++) + if (comp->sortdef[i].id == id) + return comp->sortdef[i].name; + return NULL; +} + +/* + * Check if the given @id (must exist in the associated sortdef) enabled in + * @comp. + */ +bool compare_has_id(const struct compare *comp, int id) +{ + int idx; + + if (!comp->sortdef) + return false; + + idx = -1; + for (int i = 0; comp->sortdef[i].name; i++) + if (comp->sortdef[i].id == id) + idx = i; + + if (idx < 0) + return false; + + for (int i = 0; i < comp->count; i++) + if (comp->comp[i] == comp->sortdef[idx].comp) + return true; + return false; +} + +/* + * Set up compare structure with associated sortdef from a user specified list + * of keys. + */ +int compare_setup_sort(struct compare *comp, const struct sortdef *sdef, const char *def) +{ + const char *tmp; + int id; + + tmp = def; + do { + id = compare_parse_key_to_id(comp, &tmp); + if (id == -1) { + error("unknown sort key: %s", tmp); + return -1; + } + compare_add_sort_id(comp, id); + } while (id >= 0); + + return 0; +} diff --git a/common/sort-utils.h b/common/sort-utils.h index e2a355b375..b92a8e72d9 100644 --- a/common/sort-utils.h +++ b/common/sort-utils.h @@ -17,6 +17,8 @@ #ifndef __COMMON_SORT_UTILS_H__ #define __COMMON_SORT_UTILS_H__ +#include + /* * Example: @@ -48,7 +50,7 @@ void test() { .desc = "sort by id" }, { .name = "size", .comp = (sort_cmp_t)cmp_entry_size, .desc = "sort by entry size" }, - { .name = NULL, .comp = NULL } + SORTDEF_END }; // List of keys to use for sort (e.g. from command line options) const char *sortby[] = { "size", "id" }; @@ -66,6 +68,9 @@ void test() { } */ +#define SORTDEF_END { .name = NULL, .comp = NULL } +#define SORT_MAX_KEYS 32 + typedef int (*sort_cmp_t)(const void *a, const void *b); typedef int (*sort_r_cmp_t)(const void *a, const void *b, void *data); @@ -73,10 +78,12 @@ struct sortdef { const char *name; const char *desc; sort_cmp_t comp; + /* User defined identifier of this sort key. */ + int id; }; struct compare { - sort_cmp_t comp[32]; + sort_cmp_t comp[SORT_MAX_KEYS]; unsigned long invert_map; int count; const struct sortdef *sortdef; @@ -85,5 +92,11 @@ struct compare { int compare_init(struct compare *comp, const struct sortdef *sortdef); int compare_cmp_multi(const void *a, const void *b, const struct compare *comp); int compare_add_sort_key(struct compare *comp, const char *key); +int compare_parse_key_to_id(const struct compare *comp, const char **next); +int compare_add_sort_id(struct compare *comp, int id); +int compare_key_id(const struct compare *comp, const char *key); +const char *compare_id_name(const struct compare *comp, int id); +bool compare_has_id(const struct compare *comp, int id); +int compare_setup_sort(struct compare *comp, const struct sortdef *sdef, const char *def); #endif From 29f382042b7b9236c4f34ac2fbfa3d868c015b97 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 17 Jun 2024 04:58:26 +0200 Subject: [PATCH 29/35] btrfs-progs: inspect list-chunks: better sorting, updated output Enhance the sorting capabilities of 'inspect list-chunks' to allow multiple keys. Drop the gaps, this works only for pstart and it's hard to make it work with arbitrary sort keys. Usage is printed by default, assuming this is an interesting info and even if it slows down the output (due to extra lookups) it's more convenient to print it rather than not. The options related to usage and empty were removed. Output changes: - rename Number to PNumber, meaning physical number on the device - print Devid, device number, can be also sort key Examples: btrfs inspect list-chunks /mnt btrfs inspect list-chunks --sort length,usage btrfs inspect list-chunks --sort lstart Depending on the sort key order, the output can be wild, for that the PNumber and LNumber give some hint where the chunks lie in their space. Example output: $ sudo ./btrfs inspect list-chunks --sort length,usage / Devid PNumber Type/profile PStart Length PEnd LNumber LStart Usage% ----- ------- ----------------- --------- --------- --------- ------- --------- ------ 1 7 Data/single 1.52GiB 16.00MiB 1.54GiB 69 191.68GiB 86.04 1 3 System/DUP 117.00MiB 32.00MiB 149.00MiB 40 140.17GiB 0.05 1 2 System/DUP 85.00MiB 32.00MiB 117.00MiB 39 140.17GiB 0.05 1 15 Data/single 8.04GiB 64.00MiB 8.10GiB 61 188.60GiB 94.46 1 1 Data/single 1.00MiB 84.00MiB 85.00MiB 68 191.60GiB 74.24 1 5 Metadata/DUP 341.00MiB 192.00MiB 533.00MiB 60 188.41GiB 82.58 1 4 Metadata/DUP 149.00MiB 192.00MiB 341.00MiB 59 188.41GiB 82.58 1 20 Metadata/DUP 9.29GiB 256.00MiB 9.54GiB 38 139.92GiB 57.76 1 19 Metadata/DUP 9.04GiB 256.00MiB 9.29GiB 37 139.92GiB 57.76 1 22 Metadata/DUP 9.79GiB 256.00MiB 10.04GiB 25 113.15GiB 57.93 1 21 Metadata/DUP 9.54GiB 256.00MiB 9.79GiB 24 113.15GiB 57.93 1 46 Metadata/DUP 29.29GiB 256.00MiB 29.54GiB 43 142.71GiB 62.38 Signed-off-by: David Sterba --- cmds/inspect.c | 275 ++++++++++++++++++-------------------------- common/sort-utils.h | 2 + 2 files changed, 112 insertions(+), 165 deletions(-) diff --git a/cmds/inspect.c b/cmds/inspect.c index 353ed639ee..048f6d38e4 100644 --- a/cmds/inspect.c +++ b/cmds/inspect.c @@ -718,7 +718,7 @@ struct list_chunks_entry { u64 flags; u64 lnumber; u64 used; - u32 number; + u64 pnumber; }; struct list_chunks_ctx { @@ -727,7 +727,19 @@ struct list_chunks_ctx { struct list_chunks_entry *stats; }; -static int cmp_cse_devid_start(const void *va, const void *vb) +static int cmp_cse_devid(const void *va, const void *vb) +{ + const struct list_chunks_entry *a = va; + const struct list_chunks_entry *b = vb; + + if (a->devid < b->devid) + return -1; + if (a->devid > b->devid) + return 1; + return 0; +} + +static int cmp_cse_devid_pstart(const void *va, const void *vb) { const struct list_chunks_entry *a = va; const struct list_chunks_entry *b = vb; @@ -739,38 +751,37 @@ static int cmp_cse_devid_start(const void *va, const void *vb) if (a->start < b->start) return -1; - if (a->start == b->start) { - error( - "chunks start on same offset in the same device: devid %llu start %llu", - a->devid, a->start); + if (a->start == b->start) return 0; - } return 1; } -static int cmp_cse_devid_lstart(const void *va, const void *vb) +static int cmp_cse_pstart(const void *va, const void *vb) { const struct list_chunks_entry *a = va; const struct list_chunks_entry *b = vb; - if (a->devid < b->devid) + if (a->start < b->start) return -1; - if (a->devid > b->devid) - return 1; + if (a->start == b->start) + return 0; + return 1; +} + +static int cmp_cse_lstart(const void *va, const void *vb) +{ + const struct list_chunks_entry *a = va; + const struct list_chunks_entry *b = vb; if (a->lstart < b->lstart) return -1; - if (a->lstart == b->lstart) { - error( -"chunks logically start on same offset in the same device: devid %llu start %llu", - a->devid, a->lstart); + if (a->lstart == b->lstart) return 0; - } return 1; } /* Compare entries by usage percent, descending. */ -static int cmp_cse_devid_usage(const void *va, const void *vb) +static int cmp_cse_usage(const void *va, const void *vb) { const struct list_chunks_entry *a = va; const struct list_chunks_entry *b = vb; @@ -785,24 +796,7 @@ static int cmp_cse_devid_usage(const void *va, const void *vb) return 0; } -static int cmp_cse_length_physical(const void *va, const void *vb) -{ - const struct list_chunks_entry *a = va; - const struct list_chunks_entry *b = vb; - - if (a->length < b->length) - return -1; - if (a->length > b->length) - return 1; - - if (a->start < b->start) - return -1; - if (a->start > b->start) - return 1; - return 0; -} - -static int cmp_cse_length_logical(const void *va, const void *vb) +static int cmp_cse_length(const void *va, const void *vb) { const struct list_chunks_entry *a = va; const struct list_chunks_entry *b = vb; @@ -811,156 +805,126 @@ static int cmp_cse_length_logical(const void *va, const void *vb) return -1; if (a->length > b->length) return 1; - - if (a->lstart < b->lstart) - return -1; - if (a->lstart > b->lstart) - return 1; return 0; } -static int print_list_chunks(struct list_chunks_ctx *ctx, const char* sortmode, - unsigned unit_mode, bool with_usage, bool with_empty) +static int print_list_chunks(struct list_chunks_ctx *ctx, const char *sortmode, + unsigned unit_mode) { u64 devid; struct list_chunks_entry e; struct string_table *table; + int col_count, col; int i; - int chidx; - u64 lastend; u64 number; - u32 gaps; u32 tabidx; - static const struct sortdef sortit[] = { - { .name = "pstart", .comp = (sort_cmp_t)cmp_cse_devid_start, - .desc = "sort by physical srart offset, group by device" }, - { .name = "lstart", .comp = (sort_cmp_t)cmp_cse_devid_lstart, - .desc = "sort by logical offset" }, - { .name = "usage", .comp = (sort_cmp_t)cmp_cse_devid_usage, - .desc = "sort by chunk usage" }, - { .name = "length_p", .comp = (sort_cmp_t)cmp_cse_length_physical, - .desc = "sort by lentgh, secondary by physical offset" }, - { .name = "length_l", .comp = (sort_cmp_t)cmp_cse_length_logical, - .desc = "sort by lentgh, secondary by logical offset" }, - }; enum { CHUNK_SORT_PSTART, CHUNK_SORT_LSTART, CHUNK_SORT_USAGE, - CHUNK_SORT_LENGTH_P, - CHUNK_SORT_LENGTH_L, + CHUNK_SORT_LENGTH, CHUNK_SORT_DEFAULT = CHUNK_SORT_PSTART }; - unsigned int sort_mode; + static const struct sortdef sortit[] = { + { .name = "devid", .comp = (sort_cmp_t)cmp_cse_devid, + .desc = "sort by device id (default, with pstart)", + .id = CHUNK_SORT_PSTART + }, + { .name = "pstart", .comp = (sort_cmp_t)cmp_cse_pstart, + .desc = "sort by physical start offset", + .id = CHUNK_SORT_PSTART + }, + { .name = "lstart", .comp = (sort_cmp_t)cmp_cse_lstart, + .desc = "sort by logical offset", + .id = CHUNK_SORT_LSTART + }, + { .name = "usage", .comp = (sort_cmp_t)cmp_cse_usage, + .desc = "sort by chunk usage", + .id = CHUNK_SORT_USAGE + }, + { .name = "length", .comp = (sort_cmp_t)cmp_cse_length, + .desc = "sort by lentgh", + .id = CHUNK_SORT_LENGTH + }, + SORTDEF_END + }; + const char *tmp; struct compare comp; + int id; - if (!sortmode) { - sort_mode = CHUNK_SORT_DEFAULT; - sortmode = "pstart"; - } else if (strcmp(sortmode, "pstart") == 0) { - sort_mode = CHUNK_SORT_PSTART; - } else if (strcmp(sortmode, "lstart") == 0) { - sort_mode = CHUNK_SORT_LSTART; - } else if (strcmp(sortmode, "usage") == 0) { - sort_mode = CHUNK_SORT_USAGE; - with_usage = true; - } else if (strcmp(sortmode, "length_p") == 0) { - sort_mode = CHUNK_SORT_LENGTH_P; - } else if (strcmp(sortmode, "length_l") == 0) { - sort_mode = CHUNK_SORT_LENGTH_L; - } else { - error("unknown sort mode: %s", sortmode); - exit(1); - } + compare_init(&comp, sortit); + + tmp = sortmode; + do { + id = compare_parse_key_to_id(&comp, &tmp); + if (id == -1) { + error("unknown sort key: %s", tmp); + return 1; + } + compare_add_sort_id(&comp, id); + } while (id >= 0); /* * Chunks are sorted logically as found by the ioctl, we need to sort * them once to find the physical ordering. This is the default mode. */ - qsort(ctx->stats, ctx->length, sizeof(ctx->stats[0]), cmp_cse_devid_start); + qsort(ctx->stats, ctx->length, sizeof(ctx->stats[0]), cmp_cse_devid_pstart); devid = 0; number = 0; - gaps = 0; - lastend = 0; for (i = 0; i < ctx->length; i++) { e = ctx->stats[i]; if (e.devid != devid) { devid = e.devid; number = 0; } - ctx->stats[i].number = number; - number++; - if (with_empty && sort_mode == CHUNK_SORT_PSTART && e.start != lastend) - gaps++; - lastend = e.start + e.length; + ctx->stats[i].pnumber = number++; } - compare_init(&comp, sortit); - compare_add_sort_key(&comp, sortmode); - qsort_r(ctx->stats, ctx->length, sizeof(ctx->stats[0]), (sort_r_cmp_t)compare_cmp_multi, - &comp); + /* Skip additonal sort if nothing defined by user. */ + if (comp.count > 0) + qsort_r(ctx->stats, ctx->length, sizeof(ctx->stats[0]), + (sort_r_cmp_t)compare_cmp_multi, &comp); - /* Optional usage, two rows for header and separator, gaps */ - table = table_create(7 + (int)with_usage, 2 + ctx->length + gaps); + col_count = 9; + /* Two rows for header and separator. */ + table = table_create(col_count, 2 + ctx->length); if (!table) { error_msg(ERROR_MSG_MEMORY, NULL); return 1; } - devid = 0; + /* Print header */ + col = 0; tabidx = 0; - chidx = 1; + table_printf(table, col++, tabidx, ">Devid"); + table_printf(table, col++, tabidx, ">PNumber"); + table_printf(table, col++, tabidx, ">Type/profile"); + table_printf(table, col++, tabidx, ">PStart"); + table_printf(table, col++, tabidx, ">Length"); + table_printf(table, col++, tabidx, ">PEnd"); + table_printf(table, col++, tabidx, ">LNumber"); + table_printf(table, col++, tabidx, ">LStart"); + table_printf(table, col++, tabidx, ">Usage%%"); + for (int j = 0; j < col_count; j++) + table_printf(table, j, tabidx + 1, "*-"); + + tabidx = 2; + devid = 0; for (i = 0; i < ctx->length; i++) { e = ctx->stats[i]; - /* TODO: print header and devid */ - if (e.devid != devid) { - int j; - + if (e.devid != devid) devid = e.devid; - table_printf(table, 0, tabidx, ">Number"); - table_printf(table, 1, tabidx, ">Type/profile"); - table_printf(table, 2, tabidx, ">PStart"); - table_printf(table, 3, tabidx, ">Length"); - table_printf(table, 4, tabidx, ">PEnd"); - table_printf(table, 5, tabidx, ">LNumber"); - table_printf(table, 6, tabidx, ">LStart"); - if (with_usage) { - table_printf(table, 7, tabidx, ">Usage%%"); - table_printf(table, 7, tabidx + 1, "*-"); - } - for (j = 0; j < 7; j++) - table_printf(table, j, tabidx + 1, "*-"); - - chidx = 1; - lastend = 0; - tabidx += 2; - } - if (with_empty && sort_mode == CHUNK_SORT_PSTART && e.start != lastend) { - table_printf(table, 0, tabidx, ">-"); - table_printf(table, 1, tabidx, ">%s", "empty"); - table_printf(table, 2, tabidx, ">-"); - table_printf(table, 3, tabidx, ">%s", - pretty_size_mode(e.start - lastend, unit_mode)); - table_printf(table, 4, tabidx, ">-"); - table_printf(table, 5, tabidx, ">-"); - table_printf(table, 6, tabidx, ">-"); - if (with_usage) - table_printf(table, 7, tabidx, ">-"); - tabidx++; - } - - table_printf(table, 0, tabidx, ">%llu", chidx++); - table_printf(table, 1, tabidx, ">%10s/%-6s", + col = 0; + table_printf(table, col++, tabidx, ">%llu", e.devid); + table_printf(table, col++, tabidx, ">%llu", e.pnumber + 1); + table_printf(table, col++, tabidx, ">%10s/%-6s", btrfs_group_type_str(e.flags), btrfs_group_profile_str(e.flags)); - table_printf(table, 2, tabidx, ">%s", pretty_size_mode(e.start, unit_mode)); - table_printf(table, 3, tabidx, ">%s", pretty_size_mode(e.length, unit_mode)); - table_printf(table, 4, tabidx, ">%s", pretty_size_mode(e.start + e.length, unit_mode)); - table_printf(table, 5, tabidx, ">%llu", e.lnumber + 1); - table_printf(table, 6, tabidx, ">%s", pretty_size_mode(e.lstart, unit_mode)); - if (with_usage) - table_printf(table, 7, tabidx, ">%6.2f", - (float)e.used / e.length * 100); - lastend = e.start + e.length; + table_printf(table, col++, tabidx, ">%s", pretty_size_mode(e.start, unit_mode)); + table_printf(table, col++, tabidx, ">%s", pretty_size_mode(e.length, unit_mode)); + table_printf(table, col++, tabidx, ">%s", pretty_size_mode(e.start + e.length, unit_mode)); + table_printf(table, col++, tabidx, ">%llu", e.lnumber + 1); + table_printf(table, col++, tabidx, ">%s", pretty_size_mode(e.lstart, unit_mode)); + table_printf(table, col++, tabidx, ">%6.2f", (float)e.used / e.length * 100); tabidx++; } table_dump(table); @@ -1019,8 +983,6 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd, int i; unsigned unit_mode; char *sortmode = NULL; - bool with_usage = true; - bool with_empty = true; const char *path; struct list_chunks_ctx ctx = { .length = 0, @@ -1033,15 +995,10 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd, while (1) { int c; enum { GETOPT_VAL_SORT = GETOPT_VAL_FIRST, - GETOPT_VAL_USAGE, GETOPT_VAL_NO_USAGE, GETOPT_VAL_EMPTY, GETOPT_VAL_NO_EMPTY }; static const struct option long_options[] = { {"sort", required_argument, NULL, GETOPT_VAL_SORT }, - {"usage", no_argument, NULL, GETOPT_VAL_USAGE }, - {"no-usage", no_argument, NULL, GETOPT_VAL_NO_USAGE }, - {"empty", no_argument, NULL, GETOPT_VAL_EMPTY }, - {"no-empty", no_argument, NULL, GETOPT_VAL_NO_EMPTY }, {NULL, 0, NULL, 0} }; @@ -1054,14 +1011,6 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd, free(sortmode); sortmode = strdup(optarg); break; - case GETOPT_VAL_USAGE: - case GETOPT_VAL_NO_USAGE: - with_usage = (c == GETOPT_VAL_USAGE); - break; - case GETOPT_VAL_EMPTY: - case GETOPT_VAL_NO_EMPTY: - with_empty = (c == GETOPT_VAL_EMPTY); - break; default: usage_unknown_option(cmd, argv); } @@ -1136,7 +1085,7 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd, e->lstart = sh.offset; e->length = item->length; e->flags = item->type; - e->number = -1; + e->pnumber = -1; while (devid > lnumber_size) { u64 *tmp; unsigned old_size = lnumber_size; @@ -1152,13 +1101,9 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd, lnumber = tmp; } e->lnumber = lnumber[devid]++; - if (with_usage) { - if (used == (u64)-1) - used = fill_usage(fd, sh.offset); - e->used = used; - } else { - e->used = 0; - } + if (used == (u64)-1) + used = fill_usage(fd, sh.offset); + e->used = used; ctx.length++; @@ -1184,7 +1129,7 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd, break; } - ret = print_list_chunks(&ctx, sortmode, unit_mode, with_usage, with_empty); + ret = print_list_chunks(&ctx, sortmode, unit_mode); close(fd); out: diff --git a/common/sort-utils.h b/common/sort-utils.h index b92a8e72d9..9b3f180abb 100644 --- a/common/sort-utils.h +++ b/common/sort-utils.h @@ -74,6 +74,8 @@ void test() { typedef int (*sort_cmp_t)(const void *a, const void *b); typedef int (*sort_r_cmp_t)(const void *a, const void *b, void *data); +#define SORTDEF_END { .name = NULL, .comp = NULL } + struct sortdef { const char *name; const char *desc; From ef53fbdc8e57a168e4733cd1e1a3702a552f66dd Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 18 Jun 2024 15:11:39 +0930 Subject: [PATCH 30/35] btrfs-progs: change-csum: fix the wrong metadata space reservation [BUG] 'btrfstune --csum' would always fail for a newly created btrfs: # truncate -s 1G test.img # ./mkfs.btrfs -f test.img # ./btrsftune --csum xxhash test.img WARNING: Experimental build with unstable or unfinished features WARNING: Switching checksums is experimental, do not use for valuable data! Proceed to switch checksums ERROR: failed to start transaction: Unknown error -28 ERROR: failed to start transaction: No space left on device ERROR: failed to generate new data csums: No space left on device ERROR: btrfstune failed [CAUSE] After commit e79f18a4a75f ("btrfs-progs: introduce a basic metadata free space reservation check"), btrfs_start_transaction() would check the metadata space. But at the time of introduction of csum conversion, the parameter for btrfs_start_transaction() was incorrect. The 2nd parameter is the *number* of items to be added (if we're deleting items, just pass 1). However commit 08a3bd7694b6 ("btrfs-progs: tune: add the ability to generate new data checksums") is using the item size, not the number of items to be added. This means we're passing a number 8 * nodesize times larger than the original size, no wonder we would error out with -ENOSPC. [FIX] Use proper calcuation to convert the new csum item size to number of leaves needed, and double it just in case. Pull-request: #820 Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- tune/change-csum.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tune/change-csum.c b/tune/change-csum.c index f5fc3c7f32..0e7db20f5e 100644 --- a/tune/change-csum.c +++ b/tune/change-csum.c @@ -224,14 +224,25 @@ static int generate_new_csum_range(struct btrfs_trans_handle *trans, * item. */ #define CSUM_CHANGE_BYTES_THRESHOLD (SZ_2M) + +static unsigned int calc_csum_change_nr_items(struct btrfs_fs_info *fs_info, + u16 new_csum_type) +{ + const u32 new_csum_size = btrfs_csum_type_size(new_csum_type); + const u32 csum_item_size = CSUM_CHANGE_BYTES_THRESHOLD / + fs_info->sectorsize * new_csum_size; + + return round_up(csum_item_size, fs_info->nodesize) / fs_info->nodesize * 2; +} + static int generate_new_data_csums_range(struct btrfs_fs_info *fs_info, u64 start, u16 new_csum_type) { + const unsigned int nr_items = calc_csum_change_nr_items(fs_info, new_csum_type); struct btrfs_root *csum_root = btrfs_csum_root(fs_info, 0); struct btrfs_trans_handle *trans; struct btrfs_path path = { 0 }; struct btrfs_key key; - const u32 new_csum_size = btrfs_csum_type_size(new_csum_type); void *csum_buffer; u64 converted_bytes = 0; u64 last_csum; @@ -248,9 +259,7 @@ static int generate_new_data_csums_range(struct btrfs_fs_info *fs_info, u64 star if (!csum_buffer) return -ENOMEM; - trans = btrfs_start_transaction(csum_root, - CSUM_CHANGE_BYTES_THRESHOLD / fs_info->sectorsize * - new_csum_size); + trans = btrfs_start_transaction(csum_root, nr_items); if (IS_ERR(trans)) { ret = PTR_ERR(trans); errno = -ret; @@ -306,9 +315,7 @@ static int generate_new_data_csums_range(struct btrfs_fs_info *fs_info, u64 star return -EUCLEAN; if (ret < 0) goto out; - trans = btrfs_start_transaction(csum_root, - CSUM_CHANGE_BYTES_THRESHOLD / - fs_info->sectorsize * new_csum_size); + trans = btrfs_start_transaction(csum_root, nr_items); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; From 917a7c2f416237f9983c0da4c02669e7bc9583a8 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 18 Jun 2024 16:17:18 +0930 Subject: [PATCH 31/35] btrfs-progs: tests: add test case for basic checksum conversion Test the following: - Create a btrfs with crc32c csum - Populate the filesystem - Convert the filesystem to the following csums: * xxhash * blake2 * sha256 * crc32c Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- .../064-csum-conversion-basic/test.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 tests/misc-tests/064-csum-conversion-basic/test.sh diff --git a/tests/misc-tests/064-csum-conversion-basic/test.sh b/tests/misc-tests/064-csum-conversion-basic/test.sh new file mode 100755 index 0000000000..f26059ecf7 --- /dev/null +++ b/tests/misc-tests/064-csum-conversion-basic/test.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Verify the csum conversion works as expected. + +source "$TEST_TOP/common" || exit +source "$TEST_TOP/common.convert" || exit + +check_experimental_build +setup_root_helper +prepare_test_dev + +convert_to_csum() +{ + local new_csum="$1" + + run_check "$TOP/btrfstune" --csum "$new_csum" "$TEST_DEV" + run_check "$TOP/btrfs" check --check-data-csum "$TEST_DEV" +} + +run_check_mkfs_test_dev --csum crc32c + +# We only mount the filesystem once to populate its contents, later one we +# would never mount the fs (to reduce the dependency on kernel features). +run_check_mount_test_dev +populate_fs +run_check_umount_test_dev + +convert_to_csum xxhash +convert_to_csum blake2 +convert_to_csum sha256 +convert_to_csum crc32c From 91a20bc76b7f1d9137cb1725de6b93d1ae8dab0a Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 19 Jun 2024 04:48:57 +0200 Subject: [PATCH 32/35] btrfs-progs: tests: fix fssum ASAN memory leak reports Free memory after errors in sum(), this is reported by gcc 13 on the CI. This was reproduced by: $ make D=asan TEST=019-receive-clones-on-mounted-subvol test-misc Signed-off-by: David Sterba --- tests/fssum.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/fssum.c b/tests/fssum.c index 9ae21c6e55..d3d2870a49 100644 --- a/tests/fssum.c +++ b/tests/fssum.c @@ -524,6 +524,7 @@ sum(int dirfd, int level, sum_t *dircs, char *path_prefix, char *path_in) int ret; int fd; int excl; + int error = 0; sum_file_data_t sum_file_data = flags[FLAG_STRUCTURE] ? sum_file_data_strict : sum_file_data_permissive; struct stat dir_st; @@ -542,18 +543,22 @@ sum(int dirfd, int level, sum_t *dircs, char *path_prefix, char *path_in) if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; if (entries == alloclen) { + void *tmp; + alloclen += CHUNKS; - namelist = realloc(namelist, - alloclen * sizeof(*namelist)); - if (!namelist) { - fprintf(stderr, "malloc failed\n"); - exit(-1); + tmp = realloc(namelist, alloclen * sizeof(*namelist)); + if (!tmp) { + fprintf(stderr, "realloc failed\n"); + error = 1; + goto free_namelist; } + namelist = tmp; } namelist[entries] = strdup(de->d_name); if (!namelist[entries]) { - fprintf(stderr, "malloc failed\n"); - exit(-1); + fprintf(stderr, "stdup failed\n"); + error = 1; + goto free_namelist; } ++entries; } @@ -577,13 +582,15 @@ sum(int dirfd, int level, sum_t *dircs, char *path_prefix, char *path_in) ret = fchdir(dirfd); if (ret == -1) { perror("fchdir"); - exit(-1); + error = 1; + goto free_namelist; } ret = lstat(namelist[i], &st); if (ret) { fprintf(stderr, "stat failed for %s/%s: %m\n", path_prefix, path); - exit(-1); + error = 1; + goto free_namelist; } /* We are crossing into a different subvol, skip this subtree. */ @@ -704,6 +711,14 @@ sum(int dirfd, int level, sum_t *dircs, char *path_prefix, char *path_in) next: free(path); } + +free_namelist: + closedir(d); + for (i = 0; i < entries; i++) + free(namelist[i]); + free(namelist); + if (error) + exit(-1); } int From 1dfd256a5517200aba121bb18438ea72e5f7c058 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 19 Jun 2024 04:59:54 +0200 Subject: [PATCH 33/35] btrfs-progs: ci: fix lcov warnings on unexecuted blocks There's a report on the CI after base ubuntu image update: geninfo: WARNING: /home/runner/work/btrfs-progs/btrfs-progs/common/device-scan.c:429: unexecuted block on non-branch line with non-zero hit count. Use "geninfo --rc geninfo_unexecuted_blocks=1 to set count to zero. (use "geninfo --ignore-errors gcov,gcov ..." to suppress this warning) Signed-off-by: David Sterba --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d281ee5073..5114277415 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -44,7 +44,7 @@ jobs: - name: Generate lcov results run: | lcov -c -d . -o lcov-info - genhtml -o lcov-out lcov-info + genhtml --rc geninfo_unexecuted_blocks=1 -o lcov-out lcov-info - name: Save lcov results uses: actions/upload-artifact@v4 with: From d61f8c4c95920624f365fd1c84ddb69d316e1ea3 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 19 Jun 2024 05:44:26 +0200 Subject: [PATCH 34/35] btrfs-progs: build: use thread-safe profile generation flags The CI lcov generation fails due to: Processing ./common/device-utils.gcda geninfo: ERROR: Unexpected negative count '-6' for /home/runner/work/btrfs-progs/btrfs-progs/common/device-utils.h:69. Perhaps you need to compile with '-fprofile-update=atomic (use "geninfo --ignore-errors negative ..." to bypass this error) Use a safer way to gather the profile stats. Signed-off-by: David Sterba --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 79e9b5b3ab..82dfb1b40c 100644 --- a/Makefile +++ b/Makefile @@ -314,8 +314,8 @@ ifeq ("$(origin D)", "command line") endif ifneq (,$(findstring gcov,$(D))) - DEBUG_CFLAGS_INTERNAL += -fprofile-arcs -ftest-coverage --coverage - DEBUG_LDFLAGS_INTERNAL += -fprofile-generate --coverage + DEBUG_CFLAGS_INTERNAL += -fprofile-arcs -fprofile-update=atomic -ftest-coverage --coverage + DEBUG_LDFLAGS_INTERNAL += -fprofile-generate -fprofile-update=atomic --coverage endif ifneq (,$(findstring verbose,$(D))) From 8adc70d282dd3b7dd3371655949791473cc93608 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 19 Jun 2024 05:42:33 +0200 Subject: [PATCH 35/35] btrfs-progs: tests: add fuzzing of superblock Use btrfs-sb-mod and do some arbitrary changes to super blocks. Then run 'btrfs check'. Check will fail to read the filesystem most of the time and cannot do any actual repair but must not crash. Signed-off-by: David Sterba --- Makefile | 2 +- tests/common | 2 +- tests/fuzz-tests/010-simple-sb/test.sh | 68 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100755 tests/fuzz-tests/010-simple-sb/test.sh diff --git a/Makefile b/Makefile index 82dfb1b40c..3a68671512 100644 --- a/Makefile +++ b/Makefile @@ -550,7 +550,7 @@ test-mkfs: btrfs mkfs.btrfs @echo " [TEST] mkfs-tests.sh" $(Q)bash tests/mkfs-tests.sh -test-fuzz: btrfs btrfs-image +test-fuzz: btrfs btrfs-image btrfs-sb-mod @echo " [TEST] fuzz-tests.sh" $(Q)bash tests/fuzz-tests.sh diff --git a/tests/common b/tests/common index e9b973e448..79b2b625f9 100644 --- a/tests/common +++ b/tests/common @@ -410,7 +410,7 @@ check_prereq() { # Internal tools for testing, not shipped with the package case "$1" in - btrfs-corrupt-block|btrfs-find-root|btrfs-select-super|fssum) + btrfs-corrupt-block|btrfs-find-root|btrfs-select-super|fssum|btrfs-sb-mod) if ! [ -f "$INTERNAL_BIN/$1" ]; then _fail "Failed prerequisites: $INTERNAL_BIN/$1"; fi diff --git a/tests/fuzz-tests/010-simple-sb/test.sh b/tests/fuzz-tests/010-simple-sb/test.sh new file mode 100755 index 0000000000..511c82eee5 --- /dev/null +++ b/tests/fuzz-tests/010-simple-sb/test.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Change fields in super block and do test run of 'btrfs check' + +source "$TEST_TOP/common" || exit +source "$TEST_TOP/common.convert" || exit + +check_prereq btrfs-sb-mod + +setup_root_helper +prepare_test_dev + +run_check_mkfs_test_dev +run_check_mount_test_dev +#populate_fs +generate_dataset "small" +generate_dataset "sparse" +run_check_umount_test_dev + +# See btrfs-sb-mod --help +fields=( +bytenr +flags +magic +generation +root +chunk_root +log_root +total_bytes +bytes_used +root_dir_objectid +num_devices +sectorsize +nodesize +stripesize +sys_chunk_array_size +chunk_root_generation +compat_flags +compat_ro_flags +incompat_flags +csum_type +root_level +chunk_root_level +log_root_level +cache_generation +uuid_tree_generation +dev_item.devid +dev_item.total_bytes +dev_item.bytes_used +dev_item.io_align +dev_item.io_width +dev_item.sector_size +dev_item.type +dev_item.generation +dev_item.start_offset +dev_item.dev_group +dev_item.seek_speed +dev_item.bandwidth) + +for field in "${fields[@]}"; do + for op in -1 +1 ^2 ^4 ^256 +4096 -4096 @; do + run_check $SUDO_HELPER truncate -s 0 image.test + run_check $SUDO_HELPER cp --reflink=auto --sparse=auto "$TEST_DEV" image.test + run_check $SUDO_HELPER "$TOP/btrfs-sb-mod" image.test "$field" "$op" + run_mayfail $SUDO_HELPER "$TOP/btrfs" check image.test + done +done + +run_check $SUDO_HELPER rm -f -- image.test