spdk: do not use huge pages

Signed-off-by: Alexander Indenbaum <[email protected]>
ceph · Oct 14, 2024 · d1aa692 · d1aa692
1 parent 0442968
commit d1aa692
Show file tree

Hide file tree

Showing 9 changed files with 19 additions and 98 deletions.
diff --git a/.env b/.env
@@ -20,8 +20,6 @@ TARGET_PLATFORM="linux/amd64"
 
 # Performance
 NVMEOF_NOFILE=20480 # Max number of open files (depends on number of hosts connected)
-HUGEPAGES=2048 # 4 GB
-HUGEPAGES_DIR="/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
 
 # NVMe-oF
 NVMEOF_VERSION="${VERSION}"

diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -135,15 +135,13 @@ jobs:
       matrix:
         test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "log_files", "nsid", "psk", "dhchap"]
     runs-on: ubuntu-latest
-    env:
-      HUGEPAGES: 512  # for multi gateway test, approx 256 per gateway instance
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup huge pages
+      - name: Setup
         run: |
-          make setup HUGEPAGES=$HUGEPAGES
+          make setup
 
       - name: Download container images
         uses: actions/download-artifact@v4
@@ -253,14 +251,12 @@ jobs:
       matrix:
         security_protocol: ["unsecured", "psk", "dhchap"]
     runs-on: ubuntu-latest
-    env:
-      HUGEPAGES: 512
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup huge-pages
-        run: make setup HUGEPAGES=$HUGEPAGES
+      - name: Setup
+        run: make setup
 
       - name: Download container images
         uses: actions/download-artifact@v4
@@ -358,15 +354,13 @@ jobs:
       matrix:
         integration: ["container", "embedded"]
     runs-on: ubuntu-latest
-    env:
-      HUGEPAGES: 768  # 3 spdk instances
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup huge-pages
-        run: make setup HUGEPAGES=$HUGEPAGES
+      - name: Setup
+        run: make setup
 
       - name: Download container images
         uses: actions/download-artifact@v4
@@ -540,15 +534,13 @@ jobs:
       matrix:
         test: ["sanity", "ns_lb_change", "no_subsystems", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist"]
     runs-on: ubuntu-latest
-    env:
-      HUGEPAGES: 1024  # 4 spdk instances
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup huge-pages
-        run: make setup HUGEPAGES=$HUGEPAGES
+      - name: Setup
+        run: make setup
 
       - name: Download container images
         uses: actions/download-artifact@v4

diff --git a/Dockerfile b/Dockerfile
@@ -62,9 +62,7 @@ ARG NVMEOF_NAME \
     NVMEOF_GIT_MODIFIED_FILES \
     SPDK_GIT_REPO \
     SPDK_GIT_BRANCH \
-    SPDK_GIT_COMMIT \
-    HUGEPAGES \
-    HUGEPAGES_DIR
+    SPDK_GIT_COMMIT
 
 ENV NVMEOF_VERSION="${NVMEOF_VERSION}" \
       NVMEOF_GIT_REPO="${NVMEOF_GIT_REPO}" \
@@ -76,9 +74,7 @@ ENV NVMEOF_VERSION="${NVMEOF_VERSION}" \
       NVMEOF_GIT_MODIFIED_FILES="${NVMEOF_GIT_MODIFIED_FILES}" \
       SPDK_GIT_REPO="${SPDK_GIT_REPO}" \
       SPDK_GIT_BRANCH="${SPDK_GIT_BRANCH}" \
-      SPDK_GIT_COMMIT="${SPDK_GIT_COMMIT}" \
-      HUGEPAGES="${HUGEPAGES}" \
-      HUGEPAGES_DIR="${HUGEPAGES_DIR}"
+      SPDK_GIT_COMMIT="${SPDK_GIT_COMMIT}"
 
 # Generic labels
 LABEL name="$NVMEOF_NAME" \

diff --git a/Dockerfile.spdk b/Dockerfile.spdk
@@ -177,16 +177,19 @@ ENV BDEVPERF_IO_QUEUE="128"
 # IO Block size
 ENV BDEVPERF_IO_BS="4096"
 
+# memory size in MB for DPDK
+ENV BDEVPERF_MEM_SIZE="4096"
+
 # Type of I/O pattern, see https://spdk.io/doc/bdevperf.html
 # for available types
 ENV BDEVPERF_IO_TYPE="verify"
 
 COPY --from=build /src/build/examples/bdevperf /usr/local/bin/bdevperf
 COPY --from=build /src/examples/bdev/bdevperf/bdevperf.py /usr/libexec/spdk/scripts/bdevperf.py
-RUN echo "#!/bin/sh -x"                                                 > /entrypoint
-RUN echo "bdevperf -z -r \$BDEVPERF_SOCKET \\"                         >> /entrypoint
-RUN echo "         -q \$BDEVPERF_IO_QUEUE -o \$BDEVPERF_IO_BS \\"      >> /entrypoint
-RUN echo "         -w \$BDEVPERF_IO_TYPE -t \$BDEVPERF_TEST_DURATION"  >> /entrypoint
+RUN echo "#!/bin/sh -x"                                                         > /entrypoint
+RUN echo "bdevperf --no-huge -s \$BDEVPERF_MEM_SIZE -z -r \$BDEVPERF_SOCKET \\" >> /entrypoint
+RUN echo "         -q \$BDEVPERF_IO_QUEUE -o \$BDEVPERF_IO_BS \\"               >> /entrypoint
+RUN echo "         -w \$BDEVPERF_IO_TYPE -t \$BDEVPERF_TEST_DURATION"           >> /entrypoint
 RUN chmod 755 /entrypoint
 
 ENTRYPOINT [ "/entrypoint" ]
diff --git a/Makefile b/Makefile
@@ -37,9 +37,6 @@ setup: ## Configure huge-pages (requires sudo/root password)
 	sudo mkdir -p /var/log/ceph
 	sudo chmod 0755 /var/log/ceph
 	sudo bash -c 'echo "|/usr/bin/env tee /tmp/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern'
-	sudo bash -c 'echo $(HUGEPAGES) > $(HUGEPAGES_DIR)'
-	@echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR))
-	@[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ]
 
 build pull logs down: SVC ?= ceph spdk bdevperf nvmeof nvmeof-devel nvmeof-cli discovery
 
@@ -61,7 +58,6 @@ up:
 	@$(CURDIR)/tests/ha/start_up.sh $(SCALE)
 
 clean: $(CLEAN) setup  ## Clean-up environment
-clean: override HUGEPAGES = 0
 clean:
 	/usr/bin/rm -f control/proto/gateway_pb2_grpc.py control/proto/gateway_pb2.py control/proto/gateway_pb2.pyi control/proto/monitor_pb2_grpc.py control/proto/monitor_pb2.py control/proto/monitor_pb2.pyi
 

diff --git a/README.md b/README.md
@@ -292,16 +292,6 @@ server_cert = ./server.crt
 client_cert = ./client.crt
 ```
 
-### Huge-Pages
-
-[DPDK requires hugepages](https://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#linux-gsg-hugepages) to be set up:
-
-```bash
-sh -c 'echo 4096 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages'
-```
-
-This is automatically done in the `make setup` step. The amount of hugepages can be configured with `make setup HUGEPAGES=512`.
-
 ## Development
 
 ### Set-up

diff --git a/control/grpc.py b/control/grpc.py
@@ -252,44 +252,6 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp
         config.display_environment_info(self.logger)
         self.ceph_utils = ceph_utils
         self.ceph_utils.fetch_and_display_ceph_version()
-        requested_hugepages_val = os.getenv("HUGEPAGES", "")
-        if not requested_hugepages_val:
-            self.logger.warning("Can't get requested huge pages count")
-        else:
-            requested_hugepages_val = requested_hugepages_val.strip()
-            try:
-                requested_hugepages_val = int(requested_hugepages_val)
-                self.logger.info(f"Requested huge pages count is {requested_hugepages_val}")
-            except ValueError:
-                self.logger.warning(f"Requested huge pages count value {requested_hugepages_val} is not numeric")
-                requested_hugepages_val = None
-        hugepages_file = os.getenv("HUGEPAGES_DIR", "")
-        if not hugepages_file:
-            hugepages_file = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
-            self.logger.warning("No huge pages file defined, will use /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages")
-        else:
-            hugepages_file = hugepages_file.strip()
-        if os.access(hugepages_file, os.F_OK):
-            try:
-                hugepages_val = ""
-                with open(hugepages_file) as f:
-                    hugepages_val = f.readline()
-                hugepages_val = hugepages_val.strip()
-                if hugepages_val:
-                    try:
-                        hugepages_val = int(hugepages_val)
-                        self.logger.info(f"Actual huge pages count is {hugepages_val}")
-                    except ValueError:
-                        self.logger.warning(f"Actual huge pages count value {hugepages_val} is not numeric")
-                        hugepages_val = ""
-                    if requested_hugepages_val and hugepages_val != "" and requested_hugepages_val > hugepages_val:
-                        self.logger.warning(f"The actual huge page count {hugepages_val} is smaller than the requested value of {requested_hugepages_val}")
-                else:
-                    self.logger.warning(f"Can't read actual huge pages count value from {hugepages_file}")
-            except Exception as ex:
-                self.logger.exception(f"Can't read actual huge pages count value from {hugepages_file}")
-        else:
-            self.logger.warning(f"Can't find huge pages file {hugepages_file}")
         self.config = config
         config.dump_config_file(self.logger)
         self.rpc_lock = rpc_lock

diff --git a/control/server.py b/control/server.py
@@ -390,7 +390,8 @@ def _start_spdk(self, omap_state):
         self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}")
         spdk_tgt_cmd_extra_args = self.config.get_with_default(
             "spdk", "tgt_cmd_extra_args", "")
-        cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path]
+        spdk_memsize = self.config.getint_with_default("spdk", "mem_size", 4096)
+        cmd = [spdk_tgt_path, "-u", "--no-huge", "-s", str(spdk_memsize), "-r", self.spdk_rpc_socket_path]
 
         # Add extra args from the conf file
         if spdk_tgt_cmd_extra_args:

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -40,14 +40,7 @@ services:
     image: bdevperf
     build:
       target: bdevperf
-    volumes:
-      # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
-      # https://spdk.io/doc/containers.html
-      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
-      - /dev/hugepages:/dev/hugepages
-      - /dev/vfio/vfio:/dev/vfio/vfio
     cap_add:
-      - SYS_ADMIN # huge-pages
       - CAP_SYS_NICE # RTE
       - SYS_PTRACE # gdb
     ulimits:
@@ -124,23 +117,15 @@ services:
         SPDK_GIT_REPO:
         SPDK_GIT_BRANCH:
         SPDK_GIT_COMMIT:
-        HUGEPAGES:
-        HUGEPAGES_DIR:
         CONTAINER_REGISTRY:
       labels:
         io.ceph.nvmeof:
     volumes:
-      # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
-      # https://spdk.io/doc/containers.html
-      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
-      - /dev/hugepages:/dev/hugepages
-      - /dev/vfio/vfio:/dev/vfio/vfio
       - $NVMEOF_CONFIG:/src/ceph-nvmeof.conf
       - /tmp/coredump:/tmp/coredump # core dump
       - /var/log/ceph:/var/log/ceph
       - ceph-conf:/etc/ceph:ro
     cap_add:
-      - SYS_ADMIN # huge-pages
       - CAP_SYS_NICE # RTE
       - SYS_PTRACE # gdb
     ulimits:
@@ -244,8 +229,6 @@ services:
         SPDK_GIT_REPO:
         SPDK_GIT_BRANCH:
         SPDK_GIT_COMMIT:
-        HUGEPAGES:
-        HUGEPAGES_DIR:
       labels:
         io.ceph.nvmeof:
     volumes: