s3store: Add tools for debugging bottlenecks

Dockerfile

@@ -14,6 +14,7 @@ RUN set -xe \
 COPY cmd/ ./cmd/
 COPY internal/ ./internal/
 COPY pkg/ ./pkg/
+COPY examples/ ./examples/
 
 # Get the version name and git commit as a build argument
 ARG GIT_VERSION
@@ -24,6 +25,10 @@ RUN set -xe \
         -ldflags="-X github.com/tus/tusd/cmd/tusd/cli.VersionName=${GIT_VERSION} -X github.com/tus/tusd/cmd/tusd/cli.GitCommit=${GIT_COMMIT} -X 'github.com/tus/tusd/cmd/tusd/cli.BuildDate=$(date --utc)'" \
         -o /go/bin/tusd ./cmd/tusd/main.go
 
+RUN set -xe \
+    && GOOS=linux GOARCH=amd64 go build \
+        -o /go/bin/hooks_handler ./examples/hooks/plugin/hook_handler.go
+
 # start a new stage that copies in the binary built in the previous stage
 FROM alpine:3.16.2
 WORKDIR /srv/tusd-data
@@ -39,6 +44,7 @@ RUN apk add --no-cache ca-certificates jq bash \
     && chmod +x /usr/local/share/docker-entrypoint.sh /usr/local/share/load-env.sh
 
 COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
+COPY --from=builder /go/bin/hooks_handler /usr/local/bin/hooks_handler
 
 EXPOSE 1080
 USER tusd

docker/load-tests/1_run-test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Setup traps, so our background job of monitoring the containers
+# exits, if the script is complete.
+trap "exit" INT TERM ERR
+trap "kill 0" EXIT
+
+# 1) Ensure that the containers are up-to-date
+docker compose build
+
+# 2) Start the container monitoring
+docker stats --format "{{ json . }}" > resource-usage-log.txt &
+
+# 3) Run the actual tests
+docker compose up --abort-on-container-exit

docker/load-tests/2_plot_resource_usage.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+import json
+import re
+import matplotlib.pyplot as plt
+
+snapshots = []
+
+with open("./resource-usage-log.txt") as file:
+  current_snapshot = None
+  for line in file:
+    # The lines might contain the reset characters before the actual JSON.
+    # This means that the entire resources for the current time have been
+    # written out, so we add the latest snapshot to our list and continue
+    # reading the next entries.
+    first_backet = line.find("{")
+    if first_backet == -1:
+      continue
+
+    if first_backet != 0:
+      if current_snapshot is not None:
+        snapshots.append(current_snapshot)
+
+      current_snapshot = []
+      line = line[first_backet:]
+
+    current_snapshot.append(json.loads(line))
+
+def parse_percentage(string):
+  return float(string.strip('%'))
+
+units = {"B": 1, "kB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12,
+         "KiB": 2**10, "MiB": 2**20, "GiB": 2**30, "TiB": 2**40}
+
+def parse_byte_size(size):
+  number, unit = re.findall(r'([0-9\.]+)([A-Za-z]+)', size)[0]
+  return int(float(number)*units[unit])
+
+def parse_two_bytes(string):
+  str1, str2 = string.split("/")
+  return parse_byte_size(str1), parse_byte_size(str2)
+
+s3_cpu = []
+s3_mem = []
+tusd_cpu = []
+tusd_mem = []
+tusd_net = []
+uploader_cpu = []
+uploader_mem = []
+uploader_net = []
+timestamp = []
+
+for (i, snapshot) in enumerate(snapshots):
+  a_s3_cpu = None
+  a_s3_mem = None
+  a_tusd_cpu = None
+  a_tusd_mem = None
+  a_tusd_net = None
+  a_uploader_cpu = None
+  a_uploader_mem = None
+  a_uploader_net = None
+
+  for entry in snapshot:
+    if entry["Name"] == "load-tests-tusd-1":
+      a_tusd_cpu = parse_percentage(entry["CPUPerc"])
+      a_tusd_mem = parse_two_bytes(entry["MemUsage"])[0]
+      a_tusd_net = parse_two_bytes(entry["NetIO"])[0]
+    elif entry["Name"] == "load-tests-s3-1":
+      a_s3_cpu = parse_percentage(entry["CPUPerc"])
+      a_s3_mem = parse_two_bytes(entry["MemUsage"])[0]
+    elif entry["Name"] == "load-tests-uploader-1":
+      a_uploader_cpu = parse_percentage(entry["CPUPerc"])
+      a_uploader_mem = parse_two_bytes(entry["MemUsage"])[0]
+      a_uploader_net = parse_two_bytes(entry["NetIO"])[1]
+
+  s3_cpu.append(a_s3_cpu)
+  s3_mem.append(a_s3_mem)
+  tusd_cpu.append(a_tusd_cpu)
+  tusd_mem.append(a_tusd_mem)
+  tusd_net.append(a_tusd_net)
+  uploader_cpu.append(a_uploader_cpu)
+  uploader_mem.append(a_uploader_mem)
+  uploader_net.append(a_uploader_net)
+
+  # The docker stats command is hard coded to output stats every 500ms:
+  # https://github.com/docker/cli/blob/81c68913e4c2cb058b5a9fd5972e2989d9915b2c/cli/command/container/stats.go#L223
+  timestamp.append(0.5 * i)
+
+fig, axs = plt.subplots(3, 3, sharex=True, sharey='row')
+axs[0, 0].plot(timestamp, tusd_cpu)
+axs[0, 0].set_title('tusd CPU percentage')
+axs[0, 0].set(ylabel='CPU perc', xlabel='time')
+axs[0, 1].plot(timestamp, s3_cpu)
+axs[0, 1].set_title('s3 CPU percentage')
+axs[0, 1].set(ylabel='CPU perc', xlabel='time')
+axs[0, 2].plot(timestamp, uploader_cpu)
+axs[0, 2].set_title('uploader CPU percentage')
+axs[0, 2].set(ylabel='CPU perc', xlabel='time')
+
+axs[1, 0].plot(timestamp, tusd_mem)
+axs[1, 0].set_title('tusd memory usage')
+axs[1, 0].set(ylabel='mem perc', xlabel='time')
+axs[1, 1].plot(timestamp, s3_mem)
+axs[1, 1].set_title('s3 memory usage')
+axs[1, 1].set(ylabel='mem perc', xlabel='time')
+axs[1, 2].plot(timestamp, uploader_mem)
+axs[1, 2].set_title('uploader memory usage')
+axs[1, 2].set(ylabel='mem perc', xlabel='time')
+
+axs[2, 0].plot(timestamp, tusd_net)
+axs[2, 0].set_title('tusd network input')
+axs[2, 0].set(ylabel='total volume', xlabel='time')
+axs[2, 1].axis('off')
+axs[2, 2].plot(timestamp, uploader_net)
+axs[2, 2].set_title('uploader network output')
+axs[2, 2].set(ylabel='total volume', xlabel='time')
+
+
+# Hide x labels and tick labels for top plots and y ticks for right plots.
+for ax in axs.flat:
+    ax.label_outer()
+
+plt.show()

docker/load-tests/README.md

@@ -0,0 +1,3 @@
+## Load issues with tusd & S3
+
+This 

docker/load-tests/docker-compose.yml

@@ -0,0 +1,68 @@
+version: "3.9"
+
+# TODO:
+# - Add service for monitoring tusd
+# - Add hooks
+# - Use similar configuration as api2
+
+services:
+  s3:
+    image: minio/minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    # Note: Data directory is not persistent on purpose
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: "2"
+
+  createbucket:
+    image: minio/mc
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc config host add s3 http://s3:9000 minioadmin minioadmin;
+      /usr/bin/mc mb --ignore-existing s3/tusdtest.transloadit.com;
+      sleep infinity;
+      "
+    depends_on:
+      - s3
+
+  tusd:
+    build: ../../
+    ports:
+      - "1080:1080"
+    # entrypoint: file /srv/tusdhook/hook_handler
+    entrypoint: tusd -s3-bucket "tusdtest.transloadit.com" -s3-endpoint "http://s3:9000" -hooks-plugin=/usr/local/bin/hooks_handler -hooks-enabled-events=pre-create,post-create,post-receive,post-finish -progress-hooks-interval=3000 -max-size=128849018880 -timeout=60000 -s3-disable-content-hashes=true -s3-disable-ssl=true -s3-concurrent-part-uploads=48 -s3-max-buffered-parts=1
+    environment:
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
+    depends_on:
+      - s3
+      - createbucket
+    volumes:
+      - ../../examples/hooks/plugin:/srv/tusdhook
+    deploy:
+      resources:
+        limits:
+          cpus: "2"
+
+  uploader:
+    build: ./uploader
+    #   10 MiB: 10485760
+    #  100 MiB: 104857600
+    # 1000 MiB: 1048576000
+    command: 10485760 50 /dev/shm
+    tmpfs:
+      - /dev/shm
+    depends_on:
+      - tusd
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: "1"