Add MongoDB benchmark (#23)

Dockerfile.dev

@@ -50,6 +50,18 @@ RUN apt update -y \
     bison \
     && apt clean
 
+# Install MongoDB
+RUN apt-get update && \
+    apt-get install -y gnupg curl && \
+    curl -fsSL https://pgp.mongodb.com/server-6.0.asc | gpg --dearmor -o /usr/share/keyrings/mongodb-server-6.0.gpg && \
+    echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-6.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list && \
+    apt-get update && \
+    apt-get install -y mongodb-org && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+
+
 # Base development image
 FROM ${BUILD_IMAGE} AS dev
 
@@ -86,6 +98,9 @@ RUN echo "export SRC_DIR=${SRC_DIR}" >> /root/.profile
 RUN echo "export UNAME=${UNAME}" >> /root/.profile
 RUN echo "export GID=${GID}" >> /root/.profile
 RUN echo "export LIB_PFM4_DIR=/dependencies/libpfm4" >> /root/.profile
+RUN apt-get update && \
+    apt-get install -y openjdk-11-jdk && \
+    echo 'export PATH=$PATH:/usr/lib/jvm/java-11-openjdk-amd64/bin' >> /root/.profile
 
 WORKDIR /home/${UNAME}
 

Makefile

@@ -56,6 +56,7 @@ INTERACTIVE ?= i
 # Benchmarking variables
 COLLECTION_BENCHMARK ?= faux
 BENCHMARK_DIR ?= /home/${UNAME}/kernmlops-benchmark
+YCSB_BENCHMARK_DIR ?= ${BENCHMARK_DIR}/ycsb
 
 # Provisioning variables
 PROVISIONING_USER ?= ${UNAME}
@@ -106,6 +107,24 @@ benchmark-gap:
 	-c ${KERNMLOPS_CONFIG_FILE} \
 	--benchmark gap
 
+start-mongodb:
+	mkdir -p "$(YCSB_BENCHMARK_DIR)/mongo_db"
+	@mongod --dbpath "$(YCSB_BENCHMARK_DIR)/mongo_db" --fork --logpath /var/log/mongodb.log || { echo "Error is expected, just means that the server was already running"; true; }
+
+benchmark-mongodb:
+	@${MAKE} start-mongodb
+	@python python/kernmlops collect -v \
+		-c ${KERNMLOPS_CONFIG_FILE} \
+		--benchmark mongodb
+
+load-mongodb:
+	@echo "Loading MongoDB benchmark"
+	@${MAKE} start-mongodb
+	@python $(YCSB_BENCHMARK_DIR)/ycsb-0.17.0/bin/ycsb load mongodb -s \
+		-P "$(YCSB_BENCHMARK_DIR)/ycsb-0.17.0/workloads/workloada" \
+		-p recordcount=1000000 \
+		-p mongodb.url=mongodb://localhost:27017/ycsb
+
 benchmark-linux-build:
 	@python python/kernmlops collect -v \
 	-c ${KERNMLOPS_CONFIG_FILE} \
@@ -163,6 +182,7 @@ docker:
 	@if [ ! -d "${KERNEL_DEV_MODULES_DIR}" ]; then \
 		echo "Kernel dev headers not installed: ${KERNEL_DEV_MODULES_DIR}" && exit 1; \
 	fi
+
 	@mkdir -p ${BENCHMARK_DIR}
 	@docker --context ${CONTAINER_CONTEXT} run --rm \
 	-v ${SRC_DIR}/:${CONTAINER_SRC_DIR} \
@@ -183,6 +203,13 @@ docker:
 	${IMAGE_NAME}:${VERSION} \
 	${CONTAINER_CMD} || true
 
+install-ycsb:
+	@echo "Installing ycsb..."
+	@source scripts/setup-benchmarks/install_ycsb.sh
+
+setup-mongodb:
+	@echo "Setting up storage for mongodb benchmark..."
+	@source scripts/setup-benchmarks/setup_mongodb_dir.sh
 
 # Miscellaneous commands
 clean-docker-images:

README.md

@@ -22,6 +22,12 @@ make docker-image
 # Installs gap benchmark (default)
 bash scripts/setup-benchmarks/setup-gap.sh
 
+# Installs YCSB (Yahoo! Cloud Serving Benchmark), used by MongoDB benchmark
+make install-ycsb
+
+# Installs MongoDB benchmark
+make setup-mongodb
+
 # Ensure you have installed your kernel's development headers
 # On ubuntu: apt install linux-headers-$(uname -r)
 # On redhat: dnf install kernel-devel kernel-headers
@@ -34,6 +40,11 @@ make docker
 make benchmark-gap
 # Run yaml configured data collection inside docker
 make collect
+# Run mongodb benchmark inside docker
+make docker
+make load-mongodb # Run this command only the first time you set up the MongoDB benchmark
+make benchmark-mongodb
+
 ```
 
 ## Capturing Data -> Processing in Python

defaults.yaml

@@ -11,6 +11,10 @@ benchmark_config:
   gap:
     gap_benchmark: pr
     trials: 2
+  mongodb:
+    operation_count: 1000000
+    read_proportion: 0.25
+    update_proportion: 0.75
 collector_config:
   generic:
     poll_rate: 0.5

python/kernmlops/kernmlops_benchmark/__init__.py

@@ -14,12 +14,14 @@
 )
 from kernmlops_benchmark.gap import GapBenchmark
 from kernmlops_benchmark.linux_build import LinuxBuildBenchmark
+from kernmlops_benchmark.mongodb import MongoDbBenchmark
 from kernmlops_config import ConfigBase
 
 benchmarks: Mapping[str, type[Benchmark]] = {
     FauxBenchmark.name(): FauxBenchmark,
     LinuxBuildBenchmark.name(): LinuxBuildBenchmark,
     GapBenchmark.name(): GapBenchmark,
+    MongoDbBenchmark.name(): MongoDbBenchmark,
 }
 
 BenchmarkConfig = make_dataclass(

python/kernmlops/kernmlops_benchmark/mongodb.py

@@ -0,0 +1,97 @@
+import subprocess
+from dataclasses import dataclass
+from typing import cast
+
+from data_schema import GraphEngine, demote
+from kernmlops_benchmark.benchmark import Benchmark, GenericBenchmarkConfig
+from kernmlops_benchmark.errors import (
+  BenchmarkNotInCollectionData,
+  BenchmarkNotRunningError,
+  BenchmarkRunningError,
+)
+from kernmlops_config import ConfigBase
+
+
+@dataclass(frozen=True)
+class MongoDbConfig(ConfigBase):
+  operation_count: int = 1000000
+  read_proportion: float = 0.25
+  update_proportion: float = 0.75
+
+
+class MongoDbBenchmark(Benchmark):
+
+    @classmethod
+    def name(cls) -> str:
+        return "mongodb"
+
+    @classmethod
+    def default_config(cls) -> ConfigBase:
+        return MongoDbConfig()
+
+    @classmethod
+    def from_config(cls, config: ConfigBase) -> "Benchmark":
+        generic_config = cast(GenericBenchmarkConfig, getattr(config, "generic"))
+        mongodb_config = cast(MongoDbConfig, getattr(config, cls.name()))
+        return MongoDbBenchmark(generic_config=generic_config, config=mongodb_config)
+
+    def __init__(self, *, generic_config: GenericBenchmarkConfig, config: MongoDbConfig):
+        self.generic_config = generic_config
+        self.config = config
+        self.benchmark_dir = self.generic_config.get_benchmark_dir() / "ycsb"
+        self.process: subprocess.Popen | None = None
+
+    def is_configured(self) -> bool:
+        return self.benchmark_dir.is_dir()
+
+    def setup(self) -> None:
+        if self.process is not None:
+            raise BenchmarkRunningError()
+        self.generic_config.generic_setup()
+
+    def run(self) -> None:
+        if self.process is not None:
+            raise BenchmarkRunningError()
+
+        self.process = subprocess.Popen(
+            [
+                f"{self.benchmark_dir}/ycsb-0.17.0/bin/ycsb",
+                "run",
+                "mongodb",
+                "-s",
+                "-P",
+                f"{self.benchmark_dir}/ycsb-0.17.0/workloads/workloada",
+                "-p",
+                f"operationcount={self.config.operation_count}",
+                "-p",
+                "mongodb.url=mongodb://localhost:27017/ycsb",
+                "-p",
+                f"readproportion={self.config.read_proportion}",
+                "-p",
+                f"updateproportion={self.config.update_proportion}",
+                "-p",
+                "mongodb.writeConcern=acknowledged"
+            ],
+            preexec_fn=demote(),
+            stdout=subprocess.DEVNULL,
+        )
+
+    def poll(self) -> int | None:
+        if self.process is None:
+            raise BenchmarkNotRunningError()
+        return self.process.poll()
+
+    def wait(self) -> None:
+        if self.process is None:
+            raise BenchmarkNotRunningError()
+        self.process.wait()
+
+    def kill(self) -> None:
+        if self.process is None:
+            raise BenchmarkNotRunningError()
+        self.process.terminate()
+
+    @classmethod
+    def plot_events(cls, graph_engine: GraphEngine) -> None:
+        if graph_engine.collection_data.benchmark != cls.name():
+            raise BenchmarkNotInCollectionData()

scripts/setup-benchmarks/install_ycsb.sh

@@ -0,0 +1,27 @@
+#Specify destination for benchmark
+YCSB_BENCHMARK_NAME="ycsb"
+BENCHMARK_DIR_NAME="kernmlops-benchmark"
+
+BENCHMARK_DIR="${BENCHMARK_DIR:-$HOME/$BENCHMARK_DIR_NAME}"
+YCSB_BENCHMARK_DIR="$BENCHMARK_DIR/$YCSB_BENCHMARK_NAME"
+
+if [ -d $YCSB_BENCHMARK_DIR ]; then
+    echo "Benchmark already installed at: $YCSB_BENCHMARK_DIR"
+    exit 0
+fi
+
+# Setup
+mkdir -p "$YCSB_BENCHMARK_DIR"
+
+# Download YCSB
+curl -o "$YCSB_BENCHMARK_DIR/ycsb-0.17.0.tar.gz" --location https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-0.17.0.tar.gz
+
+tar xfvz "$YCSB_BENCHMARK_DIR/ycsb-0.17.0.tar.gz" -C "$YCSB_BENCHMARK_DIR"
+
+pwd
+
+# Copy contents of ycsb_runner.py to bin/ycsb
+cp scripts/setup-benchmarks/ycsb_runner.py $YCSB_BENCHMARK_DIR/ycsb-0.17.0/bin/ycsb
+
+# Make the ycsb script executable
+chmod +x $YCSB_BENCHMARK_DIR/ycsb-0.17.0/bin/ycsb

scripts/setup-benchmarks/setup_mongodb_dir.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Define variables first
+YCSB_BENCHMARK_NAME="ycsb"
+BENCHMARK_DIR_NAME="kernmlops-benchmark"
+BENCHMARK_DIR="${BENCHMARK_DIR:-$HOME/$BENCHMARK_DIR_NAME}"
+YCSB_BENCHMARK_DIR="$BENCHMARK_DIR/$YCSB_BENCHMARK_NAME"
+
+# Then use them
+if [ -d "$YCSB_BENCHMARK_DIR/mongo_db" ]; then
+    echo "Directory $YCSB_BENCHMARK_DIR/mongo_db already exists."
+    exit 0
+fi
+mkdir -p "$YCSB_BENCHMARK_DIR/mongo_db"