Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Organize the benchmark output folder #166

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Organize the benchmark output folder #166

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
592c9df
Organize the benchmark output folder
gavinlichn Oct 18, 2024
50e64e2
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 19 additions & 17 deletions evals/benchmark/benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,19 @@ def create_run_yaml_content(service, base_url, bench_target, test_phase, num_que
# the parameter will be passed to Locust to launch fixed
# number of simulated users.
concurrency = 1
runs_list = []
index = 1
try:
load_shape = test_params["load_shape"]["name"]
load_shape_params = test_params["load_shape"]["params"][load_shape]
if load_shape_params and load_shape_params["concurrent_level"]:
if num_queries >= 0:
concurrency = max(1, num_queries // load_shape_params["concurrent_level"])
else:
concurrency = load_shape_params["concurrent_level"]
for query in num_queries:
if query >= 0:
concurrency = max(1, query // load_shape_params["concurrent_level"])
else:
concurrency = load_shape_params["concurrent_level"]
runs_list.append({"name": f"{test_phase}_{index}", "users": concurrency, "max-request": query})
index += 1
except KeyError as e:
# If the concurrent_level is not specified, load shapes should
# manage concurrency and user spawn rate by themselves.
Expand All @@ -105,7 +110,7 @@ def create_run_yaml_content(service, base_url, bench_target, test_phase, num_que
"deployment-type": test_params["deployment_type"],
"load-shape": test_params["load_shape"],
},
"runs": [{"name": test_phase, "users": concurrency, "max-request": num_queries}],
"runs": runs_list,
}
}

Expand Down Expand Up @@ -135,8 +140,8 @@ def generate_stresscli_run_yaml(
The parameters of single test case.
test_phase : str [warmup|benchmark]
Current phase of the test.
num_queries : int
The number of test requests sent to SUT
num_queries : list
The list of query numbers of test requests sent to SUT
base_url : str
The root endpoint of SUT
test_params : dict
Expand All @@ -160,9 +165,7 @@ def generate_stresscli_run_yaml(

# Dump the stresscli configuration file
service_name = case_params.get("service_name")
run_yaml_path = os.path.join(
test_params["test_output_dir"], f"run_{service_name}_{ts}_{test_phase}_{num_queries}.yaml"
)
run_yaml_path = os.path.join(test_params["test_output_dir"], f"run_{service_name}_{ts}_{test_phase}.yaml")
with open(run_yaml_path, "w") as yaml_file:
yaml.dump(stresscli_yaml, yaml_file)

Expand All @@ -180,7 +183,7 @@ def create_and_save_run_yaml(example, deployment_type, service_type, service, ba
if warm_ups is not None and warm_ups > 0:
run_yaml_paths.append(
generate_stresscli_run_yaml(
example, service_type, service, test_suite_config, "warmup", warm_ups, base_url, index
example, service_type, service, test_suite_config, "warmup", [warm_ups], base_url, index
)
)

Expand All @@ -190,17 +193,16 @@ def create_and_save_run_yaml(example, deployment_type, service_type, service, ba
# Test stop is controlled by run time
run_yaml_paths.append(
generate_stresscli_run_yaml(
example, service_type, service, test_suite_config, "benchmark", -1, base_url, index
example, service_type, service, test_suite_config, "benchmark", [-1], base_url, index
)
)
else:
# Test stop is controlled by request count
for user_queries in user_queries_lst:
run_yaml_paths.append(
generate_stresscli_run_yaml(
example, service_type, service, test_suite_config, "benchmark", user_queries, base_url, index
)
run_yaml_paths.append(
generate_stresscli_run_yaml(
example, service_type, service, test_suite_config, "benchmark", user_queries_lst, base_url, index
)
)

return run_yaml_paths

Expand Down