From 629823928a496d8596e0c0e7026595a890939014 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 26 Aug 2024 00:22:45 +0200 Subject: [PATCH] feat: test many files --- .github/workflows/netsim_integration.yml | 2 +- fixtures/bulk_files_test_setup.sh | 9 + fixtures/generate_files.sh | 27 +++ netsim/setup.sh | 3 + netsim/sims/iroh/iroh_many_files.json | 237 +++++++++++++++++++++++ 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100755 fixtures/bulk_files_test_setup.sh create mode 100755 fixtures/generate_files.sh create mode 100644 netsim/sims/iroh/iroh_many_files.json diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index efd6f9c..a6feb76 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -58,7 +58,7 @@ jobs: cd netsim sudo kill -9 $(pgrep ovs) sudo mn --clean - sudo python3 main.py --integration sims/standard/iroh.json + sudo python3 main.py --integration sims/iroh/iroh_many_files.json - name: Setup Environment (PR) if: ${{ github.event_name == 'pull_request' }} diff --git a/fixtures/bulk_files_test_setup.sh b/fixtures/bulk_files_test_setup.sh new file mode 100755 index 0000000..6760bb5 --- /dev/null +++ b/fixtures/bulk_files_test_setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +./generate_files.sh 1000 10000 bulk_1k_x_10k +./generate_files.sh 5000 10000 bulk_5k_x_10k +./generate_files.sh 1000 1000000 bulk_1k_x_1m +./generate_files.sh 1000 10000 bulk_mix #1kx10k +./generate_files.sh 1000 1000000 bulk_mix #1kx1m +./generate_files.sh 100 10000000 bulk_mix #100x10m +./generate_files.sh 2 1000000000 bulk_mix #2x1g \ No newline at end of file diff --git a/fixtures/generate_files.sh b/fixtures/generate_files.sh new file mode 100755 index 0000000..b021776 --- /dev/null +++ b/fixtures/generate_files.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Check if the correct number of arguments are provided +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Assign arguments to variables +NUMBER_OF_FILES=$1 +FILE_SIZE=$2 +OUTPUT_FOLDER=$3 + +# Create the output folder if it doesn't exist +mkdir -p "$OUTPUT_FOLDER" + +# Generate a random 4-character string +RANDOM_STRING=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 4) + +# Generate the files +for (( i=1; i<=$NUMBER_OF_FILES; i++ )) +do + # Generate a file with the specified size + dd if=/dev/urandom of="$OUTPUT_FOLDER/file_${RANDOM_STRING}_$i.bin" bs=$FILE_SIZE count=1 +done + +echo "Generated $NUMBER_OF_FILES files of size $FILE_SIZE bytes in $OUTPUT_FOLDER with prefix file_$RANDOM_STRING" \ No newline at end of file diff --git a/netsim/setup.sh b/netsim/setup.sh index 679e2ed..1e8677a 100755 --- a/netsim/setup.sh +++ b/netsim/setup.sh @@ -29,3 +29,6 @@ cp ../../fixtures/direct_relay.cfg direct_relay.cfg cp ../../fixtures/relay.direct.config.toml relay.direct.config.toml cp ../../fixtures/1MB.bin 1MB.bin cp ../../fixtures/hello.bin hello.bin +cp ../../fixtures/generate_files.sh generate_files.sh +cp ../../fixtures/bulk_files_test_setup.sh bulk_files_test_setup.sh +./bulk_files_test_setup.sh \ No newline at end of file diff --git a/netsim/sims/iroh/iroh_many_files.json b/netsim/sims/iroh/iroh_many_files.json new file mode 100644 index 0000000..44aa4d9 --- /dev/null +++ b/netsim/sims/iroh/iroh_many_files.json @@ -0,0 +1,237 @@ +{ + "name": "iroh_mf", + "cases": [ + { + "name": "1_to_1_1k_10k", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_1k_x_10k", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 1, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_3_1k_10k", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_1k_x_10k", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 3, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_1_5k_10k", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_5k_x_10k", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 1, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_3_5k_10k", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_5k_x_10k", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 3, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_1_1k_1m", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_1k_x_1m", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 1, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_3_5k_10k", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_1k_x_1m", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 3, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_1_mix", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_mix", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 1, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + }, + { + "name": "1_to_3_mix", + "description": "", + "nodes": [ + { + "name": "iroh_srv", + "count": 1, + "cmd": "./bins/iroh start --add data/bulk_mix", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_ticket" + }, + { + "name": "iroh_get", + "count": 3, + "cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null", + "type": "public", + "connect": { + "strategy": "params", + "node": "iroh_srv" + }, + "process": "short", + "parser": "iroh_1gb" + } + ] + } + ] +}