Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Benchmark test] Integrate the feature of increasing the data corpus size #3826

Closed
wants to merge 5 commits into from
Closed

[Benchmark test] Integrate the feature of increasing the data corpus size #3826

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3e19afe
Comment out role_params to make benchmark test run locally
Jul 5, 2023
bd5aa3d
Add ability of expanding workload data size for benchmark test
Jul 6, 2023
42ae512
Modify the url for minimal distribution
Jul 28, 2023
8544561
Correct the location for a quote and adjust a if statement
Jul 28, 2023
c827e87
Revert changes in benchmark_test_cluster.py
Jul 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/test_workflow/benchmark_test/benchmark_args.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class BenchmarkArgs:
target_hosts: str
capture_node_stat: bool
logging_level: int
expanded_data_size: int

def __init__(self) -> None:
parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle")
Expand Down Expand Up @@ -93,6 +94,8 @@ def __init__(self) -> None:
help="Enable opensearch-benchmark to capture node stat metrics such as cpu, mem, jvm etc as well.")
parser.add_argument("-v", "--verbose", help="Show more verbose output.", action="store_const", default=logging.INFO,
const=logging.DEBUG, dest="logging_level")
parser.add_argument("--expanded-data-size", dest="expanded_data_size",
help="increasing workload data size")

args = parser.parse_args()
self.bundle_manifest = args.bundle_manifest
Expand Down Expand Up @@ -120,3 +123,4 @@ def __init__(self) -> None:
self.use_50_percent_heap = args.use_50_percent_heap
self.capture_node_stat = args.capture_node_stat
self.logging_level = args.logging_level
self.expanded_data_size = args.expanded_data_size if args.expanded_data_size else None
17 changes: 15 additions & 2 deletions src/test_workflow/benchmark_test/benchmark_test_suite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,18 @@ def __init__(
self.command = 'docker run --rm'
if args.benchmark_config:
self.command += f" -v {args.benchmark_config}:/opensearch-benchmark/.benchmark/benchmark.ini"
self.command += f" opensearchproject/opensearch-benchmark:latest execute-test --workload={self.args.workload} " \
f"--pipeline=benchmark-only --target-hosts={endpoint}"

if args.expanded_data_size is None:
self.command += f" opensearchproject/opensearch-benchmark:latest execute-test --workload={self.args.workload} " \
f"--pipeline=benchmark-only --target-hosts={endpoint}"
else:
self.command += " --entrypoint /bin/bash opensearchproject/opensearch-benchmark:latest -c \"" \
f"opensearch-benchmark execute-test --workload={self.args.workload} " \
f"--pipeline=benchmark-only --target-hosts={endpoint} " \
f"--include-tasks=index-append --workload-params='ingest_percentage:0.1'; " \
f"expand-data-corpus.py -c {args.expanded_data_size}; " \
f"opensearch-benchmark execute-test --workload={self.args.workload} " \
f"--pipeline=benchmark-only --target-hosts={endpoint}"

if args.workload_params:
logging.info(f"Workload Params are {args.workload_params}")
Expand All @@ -56,5 +66,8 @@ def execute(self) -> None:
self.command += ' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"'
else:
self.command += ' --client-options="timeout:300"'

if self.args.expanded_data_size is not None:
self.command += "\""
logging.info(f"Executing {self.command}")
subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True)
Loading