From c0b0018e97ca790e872d7734a2c216b5c8814aab Mon Sep 17 00:00:00 2001 From: Alexander Indenbaum Date: Mon, 14 Oct 2024 10:23:08 +0000 Subject: [PATCH] spdk: do not use huge pages Signed-off-by: Alexander Indenbaum --- .env | 2 -- .github/workflows/build-container.yml | 24 ++++++----------- Dockerfile | 8 ++---- Dockerfile.spdk | 11 +++++--- Makefile | 4 --- README.md | 10 ------- control/grpc.py | 38 --------------------------- control/server.py | 3 ++- docker-compose.yaml | 17 ------------ 9 files changed, 19 insertions(+), 98 deletions(-) diff --git a/.env b/.env index f1a35446..1a1b7e5e 100644 --- a/.env +++ b/.env @@ -20,8 +20,6 @@ TARGET_PLATFORM="linux/amd64" # Performance NVMEOF_NOFILE=20480 # Max number of open files (depends on number of hosts connected) -HUGEPAGES=2048 # 4 GB -HUGEPAGES_DIR="/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" # NVMe-oF NVMEOF_VERSION="${VERSION}" diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index c8843ca9..74dfa719 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -135,15 +135,13 @@ jobs: matrix: test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "log_files", "nsid", "psk", "dhchap"] runs-on: ubuntu-latest - env: - HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup huge pages + - name: Setup run: | - make setup HUGEPAGES=$HUGEPAGES + make setup - name: Download container images uses: actions/download-artifact@v4 @@ -253,14 +251,12 @@ jobs: matrix: security_protocol: ["unsecured", "psk", "dhchap"] runs-on: ubuntu-latest - env: - HUGEPAGES: 512 steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup huge-pages - run: make setup HUGEPAGES=$HUGEPAGES + - name: Setup + run: make setup - name: Download container images uses: actions/download-artifact@v4 @@ -358,15 +354,13 @@ jobs: matrix: integration: ["container", "embedded"] runs-on: ubuntu-latest - env: - HUGEPAGES: 768 # 3 spdk instances steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup huge-pages - run: make setup HUGEPAGES=$HUGEPAGES + - name: Setup + run: make setup - name: Download container images uses: actions/download-artifact@v4 @@ -540,15 +534,13 @@ jobs: matrix: test: ["sanity", "ns_lb_change", "no_subsystems", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit"] runs-on: ubuntu-latest - env: - HUGEPAGES: 1024 # 4 spdk instances steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup huge-pages - run: make setup HUGEPAGES=$HUGEPAGES + - name: Setup + run: make setup - name: Download container images uses: actions/download-artifact@v4 diff --git a/Dockerfile b/Dockerfile index 7157bddc..0ba32a39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,9 +62,7 @@ ARG NVMEOF_NAME \ NVMEOF_GIT_MODIFIED_FILES \ SPDK_GIT_REPO \ SPDK_GIT_BRANCH \ - SPDK_GIT_COMMIT \ - HUGEPAGES \ - HUGEPAGES_DIR + SPDK_GIT_COMMIT ENV NVMEOF_VERSION="${NVMEOF_VERSION}" \ NVMEOF_GIT_REPO="${NVMEOF_GIT_REPO}" \ @@ -76,9 +74,7 @@ ENV NVMEOF_VERSION="${NVMEOF_VERSION}" \ NVMEOF_GIT_MODIFIED_FILES="${NVMEOF_GIT_MODIFIED_FILES}" \ SPDK_GIT_REPO="${SPDK_GIT_REPO}" \ SPDK_GIT_BRANCH="${SPDK_GIT_BRANCH}" \ - SPDK_GIT_COMMIT="${SPDK_GIT_COMMIT}" \ - HUGEPAGES="${HUGEPAGES}" \ - HUGEPAGES_DIR="${HUGEPAGES_DIR}" + SPDK_GIT_COMMIT="${SPDK_GIT_COMMIT}" # Generic labels LABEL name="$NVMEOF_NAME" \ diff --git a/Dockerfile.spdk b/Dockerfile.spdk index bde84ada..8dcb7f36 100644 --- a/Dockerfile.spdk +++ b/Dockerfile.spdk @@ -177,16 +177,19 @@ ENV BDEVPERF_IO_QUEUE="128" # IO Block size ENV BDEVPERF_IO_BS="4096" +# memory size in MB for DPDK +ENV BDEVPERF_MEM_SIZE="4096" + # Type of I/O pattern, see https://spdk.io/doc/bdevperf.html # for available types ENV BDEVPERF_IO_TYPE="verify" COPY --from=build /src/build/examples/bdevperf /usr/local/bin/bdevperf COPY --from=build /src/examples/bdev/bdevperf/bdevperf.py /usr/libexec/spdk/scripts/bdevperf.py -RUN echo "#!/bin/sh -x" > /entrypoint -RUN echo "bdevperf -z -r \$BDEVPERF_SOCKET \\" >> /entrypoint -RUN echo " -q \$BDEVPERF_IO_QUEUE -o \$BDEVPERF_IO_BS \\" >> /entrypoint -RUN echo " -w \$BDEVPERF_IO_TYPE -t \$BDEVPERF_TEST_DURATION" >> /entrypoint +RUN echo "#!/bin/sh -x" > /entrypoint +RUN echo "bdevperf --no-huge -s \$BDEVPERF_MEM_SIZE -z -r \$BDEVPERF_SOCKET \\" >> /entrypoint +RUN echo " -q \$BDEVPERF_IO_QUEUE -o \$BDEVPERF_IO_BS \\" >> /entrypoint +RUN echo " -w \$BDEVPERF_IO_TYPE -t \$BDEVPERF_TEST_DURATION" >> /entrypoint RUN chmod 755 /entrypoint ENTRYPOINT [ "/entrypoint" ] diff --git a/Makefile b/Makefile index 8afae3e1..cb7c25b3 100644 --- a/Makefile +++ b/Makefile @@ -37,9 +37,6 @@ setup: ## Configure huge-pages (requires sudo/root password) sudo mkdir -p /var/log/ceph sudo chmod 0755 /var/log/ceph sudo bash -c 'echo "|/usr/bin/env tee /tmp/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern' - sudo bash -c 'echo $(HUGEPAGES) > $(HUGEPAGES_DIR)' - @echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR)) - @[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ] build pull logs down: SVC ?= ceph spdk bdevperf nvmeof nvmeof-devel nvmeof-cli discovery @@ -61,7 +58,6 @@ up: @$(CURDIR)/tests/ha/start_up.sh $(SCALE) clean: $(CLEAN) setup ## Clean-up environment -clean: override HUGEPAGES = 0 clean: /usr/bin/rm -f control/proto/gateway_pb2_grpc.py control/proto/gateway_pb2.py control/proto/gateway_pb2.pyi control/proto/monitor_pb2_grpc.py control/proto/monitor_pb2.py control/proto/monitor_pb2.pyi diff --git a/README.md b/README.md index d1d419fa..13041f6a 100644 --- a/README.md +++ b/README.md @@ -292,16 +292,6 @@ server_cert = ./server.crt client_cert = ./client.crt ``` -### Huge-Pages - -[DPDK requires hugepages](https://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#linux-gsg-hugepages) to be set up: - -```bash -sh -c 'echo 4096 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages' -``` - -This is automatically done in the `make setup` step. The amount of hugepages can be configured with `make setup HUGEPAGES=512`. - ## Development ### Set-up diff --git a/control/grpc.py b/control/grpc.py index 5d7b0b46..269a5497 100644 --- a/control/grpc.py +++ b/control/grpc.py @@ -252,44 +252,6 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp config.display_environment_info(self.logger) self.ceph_utils = ceph_utils self.ceph_utils.fetch_and_display_ceph_version() - requested_hugepages_val = os.getenv("HUGEPAGES", "") - if not requested_hugepages_val: - self.logger.warning("Can't get requested huge pages count") - else: - requested_hugepages_val = requested_hugepages_val.strip() - try: - requested_hugepages_val = int(requested_hugepages_val) - self.logger.info(f"Requested huge pages count is {requested_hugepages_val}") - except ValueError: - self.logger.warning(f"Requested huge pages count value {requested_hugepages_val} is not numeric") - requested_hugepages_val = None - hugepages_file = os.getenv("HUGEPAGES_DIR", "") - if not hugepages_file: - hugepages_file = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" - self.logger.warning("No huge pages file defined, will use /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages") - else: - hugepages_file = hugepages_file.strip() - if os.access(hugepages_file, os.F_OK): - try: - hugepages_val = "" - with open(hugepages_file) as f: - hugepages_val = f.readline() - hugepages_val = hugepages_val.strip() - if hugepages_val: - try: - hugepages_val = int(hugepages_val) - self.logger.info(f"Actual huge pages count is {hugepages_val}") - except ValueError: - self.logger.warning(f"Actual huge pages count value {hugepages_val} is not numeric") - hugepages_val = "" - if requested_hugepages_val and hugepages_val != "" and requested_hugepages_val > hugepages_val: - self.logger.warning(f"The actual huge page count {hugepages_val} is smaller than the requested value of {requested_hugepages_val}") - else: - self.logger.warning(f"Can't read actual huge pages count value from {hugepages_file}") - except Exception as ex: - self.logger.exception(f"Can't read actual huge pages count value from {hugepages_file}") - else: - self.logger.warning(f"Can't find huge pages file {hugepages_file}") self.config = config config.dump_config_file(self.logger) self.rpc_lock = rpc_lock diff --git a/control/server.py b/control/server.py index 9aac809b..793ad233 100644 --- a/control/server.py +++ b/control/server.py @@ -475,7 +475,8 @@ def _start_spdk(self, omap_state): self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}") spdk_tgt_cmd_extra_args = self.config.get_with_default( "spdk", "tgt_cmd_extra_args", "") - cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path] + spdk_memsize = self.config.getint_with_default("spdk", "mem_size", 4096) + cmd = [spdk_tgt_path, "-u", "--no-huge", "-s", str(spdk_memsize), "-r", self.spdk_rpc_socket_path] # Add extra args from the conf file if spdk_tgt_cmd_extra_args: diff --git a/docker-compose.yaml b/docker-compose.yaml index f480d7fa..7f5ed909 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -40,14 +40,7 @@ services: image: bdevperf build: target: bdevperf - volumes: - # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' - # https://spdk.io/doc/containers.html - # TODO: Pending of https://github.com/spdk/spdk/issues/2973 - - /dev/hugepages:/dev/hugepages - - /dev/vfio/vfio:/dev/vfio/vfio cap_add: - - SYS_ADMIN # huge-pages - CAP_SYS_NICE # RTE - SYS_PTRACE # gdb ulimits: @@ -124,23 +117,15 @@ services: SPDK_GIT_REPO: SPDK_GIT_BRANCH: SPDK_GIT_COMMIT: - HUGEPAGES: - HUGEPAGES_DIR: CONTAINER_REGISTRY: labels: io.ceph.nvmeof: volumes: - # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' - # https://spdk.io/doc/containers.html - # TODO: Pending of https://github.com/spdk/spdk/issues/2973 - - /dev/hugepages:/dev/hugepages - - /dev/vfio/vfio:/dev/vfio/vfio - $NVMEOF_CONFIG:/src/ceph-nvmeof.conf - /tmp/coredump:/tmp/coredump # core dump - /var/log/ceph:/var/log/ceph - ceph-conf:/etc/ceph:ro cap_add: - - SYS_ADMIN # huge-pages - CAP_SYS_NICE # RTE - SYS_PTRACE # gdb ulimits: @@ -244,8 +229,6 @@ services: SPDK_GIT_REPO: SPDK_GIT_BRANCH: SPDK_GIT_COMMIT: - HUGEPAGES: - HUGEPAGES_DIR: labels: io.ceph.nvmeof: volumes: