Skip to content

Commit

Permalink
feat: test many files
Browse files Browse the repository at this point in the history
  • Loading branch information
@Arqu
Arqu committed Aug 25, 2024
1 parent 3056617 commit 6298239
Show file tree
Hide file tree
Showing 5 changed files with 277 additions and 1 deletion.
2 changes: 1 addition & 1 deletion .github/workflows/netsim_integration.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
cd netsim
sudo kill -9 $(pgrep ovs)
sudo mn --clean
sudo python3 main.py --integration sims/standard/iroh.json
sudo python3 main.py --integration sims/iroh/iroh_many_files.json
- name: Setup Environment (PR)
if: ${{ github.event_name == 'pull_request' }}
Expand Down
9 changes: 9 additions & 0 deletions fixtures/bulk_files_test_setup.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/bash

./generate_files.sh 1000 10000 bulk_1k_x_10k
./generate_files.sh 5000 10000 bulk_5k_x_10k
./generate_files.sh 1000 1000000 bulk_1k_x_1m
./generate_files.sh 1000 10000 bulk_mix #1kx10k
./generate_files.sh 1000 1000000 bulk_mix #1kx1m
./generate_files.sh 100 10000000 bulk_mix #100x10m
./generate_files.sh 2 1000000000 bulk_mix #2x1g
27 changes: 27 additions & 0 deletions fixtures/generate_files.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash

# Check if the correct number of arguments are provided
if [ "$#" -ne 3 ]; then
echo "Usage: $0 <number_of_files> <file_size_in_bytes> <output_folder>"
exit 1
fi

# Assign arguments to variables
NUMBER_OF_FILES=$1
FILE_SIZE=$2
OUTPUT_FOLDER=$3

# Create the output folder if it doesn't exist
mkdir -p "$OUTPUT_FOLDER"

# Generate a random 4-character string
RANDOM_STRING=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 4)

# Generate the files
for (( i=1; i<=$NUMBER_OF_FILES; i++ ))
do
# Generate a file with the specified size
dd if=/dev/urandom of="$OUTPUT_FOLDER/file_${RANDOM_STRING}_$i.bin" bs=$FILE_SIZE count=1
done

echo "Generated $NUMBER_OF_FILES files of size $FILE_SIZE bytes in $OUTPUT_FOLDER with prefix file_$RANDOM_STRING"
3 changes: 3 additions & 0 deletions netsim/setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ cp ../../fixtures/direct_relay.cfg direct_relay.cfg
cp ../../fixtures/relay.direct.config.toml relay.direct.config.toml
cp ../../fixtures/1MB.bin 1MB.bin
cp ../../fixtures/hello.bin hello.bin
cp ../../fixtures/generate_files.sh generate_files.sh
cp ../../fixtures/bulk_files_test_setup.sh bulk_files_test_setup.sh
./bulk_files_test_setup.sh
237 changes: 237 additions & 0 deletions netsim/sims/iroh/iroh_many_files.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
{
"name": "iroh_mf",
"cases": [
{
"name": "1_to_1_1k_10k",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_1k_x_10k",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 1,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_3_1k_10k",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_1k_x_10k",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 3,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_1_5k_10k",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_5k_x_10k",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 1,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_3_5k_10k",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_5k_x_10k",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 3,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_1_1k_1m",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_1k_x_1m",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 1,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_3_5k_10k",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_1k_x_1m",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 3,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_1_mix",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_mix",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 1,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
},
{
"name": "1_to_3_mix",
"description": "",
"nodes": [
{
"name": "iroh_srv",
"count": 1,
"cmd": "./bins/iroh start --add data/bulk_mix",
"type": "public",
"wait": 10,
"connect": {
"strategy": "none"
},
"param_parser": "iroh_ticket"
},
{
"name": "iroh_get",
"count": 3,
"cmd": "time ./bins/iroh blobs get --start %s --out STDOUT > /dev/null",
"type": "public",
"connect": {
"strategy": "params",
"node": "iroh_srv"
},
"process": "short",
"parser": "iroh_1gb"
}
]
}
]
}

0 comments on commit 6298239

Please sign in to comment.