From 2db86799f856b11732d13d932ea5e76be3616353 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 30 Aug 2024 15:07:19 +0200 Subject: [PATCH 01/14] Improve leak testing script Signed-off-by: Juanjo Alvarez --- scripts/iast/test_leak_functions.py | 85 ++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/scripts/iast/test_leak_functions.py b/scripts/iast/test_leak_functions.py index 6e276520367..db7cd69c52d 100644 --- a/scripts/iast/test_leak_functions.py +++ b/scripts/iast/test_leak_functions.py @@ -3,33 +3,80 @@ import sys from mod_leak_functions import test_doit +from ddtrace.appsec._iast._taint_tracking import create_context, is_pyobject_tainted, reset_context +import argparse -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import reset_context +def parse_arguments(): + parser = argparse.ArgumentParser(description="Memory leak test script.") + parser.add_argument("--mode", choices=["ci", "console"], default="console", help="Mode of operation.") + parser.add_argument("--iterations", type=int, default=100000, help="Number of iterations.") + parser.add_argument( + "--failPercent", type=float, default=2.0, help="Failure threshold for memory increase percentage." + ) + parser.add_argument("--printEvery", type=int, help="Print status every N iterations.") + parser.add_argument( + "--graph", type=lambda x: (str(x).lower() == "true"), default=True, help="Enable ASCII graph output." + ) + + args = parser.parse_args() + + # Set default for printEvery if not provided + if args.printEvery is None: + args.printEvery = 1000 if args.mode == "ci" else 250 + + return args + + +def test_iast_leaks(): + args = parse_arguments() -def test_main(): try: - rounds = int(sys.argv[1]) - except ValueError: - rounds = 1 - print("Test %d rounds" % rounds) - for i in range(rounds): - try: + rss_list = [] + half_iterations = args.iterations // 2 + print("Test %d iterations" % args.iterations) + current_rss = 0 + half_rss = 0 + + for i in range(args.iterations): create_context() result = test_doit() # noqa: F841 assert is_pyobject_tainted(result) reset_context() - except KeyboardInterrupt: - print("Control-C stopped at %d rounds" % i) - break - if i % 250 == 0: - print("Round %d Max RSS: " % i, end="") - print(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024) - print("Round %d Max RSS: " % rounds, end="") - print(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024) + + if i == half_iterations: + half_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + + current_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + if args.graph: + rss_list.append(int(current_rss)) + + if i % args.printEvery == 0: + print(f"Round {i} Max RSS: {current_rss}") + + final_rss = current_rss + + print(f"Round {args.iterations} Max RSS: {final_rss}") + + if args.graph: + # TODO: write an ascii graph + pass + + percent_increase = ((final_rss - half_rss) / half_rss) * 100 + if percent_increase > args.failPercent: + print( + f"Failed: memory increase from half-point ({half_iterations} iterations) is {percent_increase:.2f}% which is greater than {args.failPercent}%" + ) + return 1 + else: + print( + f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} iterations) which is less than {args.failPercent}%" + ) + return 0 + + except KeyboardInterrupt: + print("Test interrupted.") if __name__ == "__main__": - test_main() + sys.exit(test_iast_leaks()) From bc219e47e4ea77ef5aac6c7b950f8b73fdd3cded Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 30 Aug 2024 18:47:52 +0200 Subject: [PATCH 02/14] Add new IAST memory leak test Signed-off-by: Juanjo Alvarez --- scripts/iast/mod_leak_functions.py | 17 ++--- scripts/iast/test_leak_functions.py | 72 +++++++++---------- .../iast_memcheck/test_iast_mem_check.py | 7 ++ 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/scripts/iast/mod_leak_functions.py b/scripts/iast/mod_leak_functions.py index d662606929a..89405aad5d6 100644 --- a/scripts/iast/mod_leak_functions.py +++ b/scripts/iast/mod_leak_functions.py @@ -4,23 +4,19 @@ import requests -from ddtrace.appsec._iast._utils import _is_iast_enabled +from tests.utils import override_env -if _is_iast_enabled(): +with override_env({"DD_IAST_ENABLED": "True"}): from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import taint_pyobject def test_doit(): origin_string1 = "hiroot" - - if _is_iast_enabled(): - tainted_string_2 = taint_pyobject( - pyobject="1234", source_name="abcdefghijk", source_value="1234", source_origin=OriginType.PARAMETER - ) - else: - tainted_string_2 = "1234" + tainted_string_2 = taint_pyobject( + pyobject="1234", source_name="abcdefghijk", source_value="1234", source_origin=OriginType.PARAMETER + ) string1 = str(origin_string1) # String with 1 propagation range string2 = str(tainted_string_2) # String with 1 propagation range @@ -71,6 +67,5 @@ def test_doit(): string19 = os.path.normcase(string18) # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted string20 = os.path.splitdrive(string19)[1] # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - expected = "notainted_HIROOT1234-HIROOT123_notainted" # noqa: F841 - # assert string20 == expected + # expected = "notainted_HIROOT1234-HIROOT123_notainted" # noqa: F841 return string20 diff --git a/scripts/iast/test_leak_functions.py b/scripts/iast/test_leak_functions.py index db7cd69c52d..f1b94c90f9a 100644 --- a/scripts/iast/test_leak_functions.py +++ b/scripts/iast/test_leak_functions.py @@ -1,46 +1,46 @@ -import ddtrace.auto # noqa: F401 # isort: skip +import argparse import resource import sys -from mod_leak_functions import test_doit -from ddtrace.appsec._iast._taint_tracking import create_context, is_pyobject_tainted, reset_context -import argparse +from tests.appsec.iast.aspects.conftest import _iast_patched_module +from tests.utils import override_env + + +with override_env({"DD_IAST_ENABLED": "True"}): + from ddtrace.appsec._iast._taint_tracking import create_context + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking import reset_context def parse_arguments(): parser = argparse.ArgumentParser(description="Memory leak test script.") - parser.add_argument("--mode", choices=["ci", "console"], default="console", help="Mode of operation.") parser.add_argument("--iterations", type=int, default=100000, help="Number of iterations.") parser.add_argument( - "--failPercent", type=float, default=2.0, help="Failure threshold for memory increase percentage." + "--fail_percent", type=float, default=2.0, help="Failure threshold for memory increase percentage." ) - parser.add_argument("--printEvery", type=int, help="Print status every N iterations.") - parser.add_argument( - "--graph", type=lambda x: (str(x).lower() == "true"), default=True, help="Enable ASCII graph output." - ) - - args = parser.parse_args() + parser.add_argument("--print_every", type=int, default=250, help="Print status every N iterations.") + return parser.parse_args() - # Set default for printEvery if not provided - if args.printEvery is None: - args.printEvery = 1000 if args.mode == "ci" else 250 - - return args - - -def test_iast_leaks(): - args = parse_arguments() +def test_iast_leaks(iterations: int, fail_percent: float, print_every: int): + if iterations < 100000: + print( + "Warning: running with %d iterations. At least 100.000 are recommended to stabilize the RSS info" + % iterations + ) try: - rss_list = [] - half_iterations = args.iterations // 2 - print("Test %d iterations" % args.iterations) + half_iterations = iterations // 2 + print("Test %d iterations" % iterations) current_rss = 0 half_rss = 0 - for i in range(args.iterations): + mod = _iast_patched_module("scripts.iast.mod_leak_functions") + test_doit = mod.test_doit + + for i in range(iterations): create_context() result = test_doit() # noqa: F841 + assert result == "notainted_HIROOT1234-HIROOT123_notainted" # noqa: F841 assert is_pyobject_tainted(result) reset_context() @@ -48,29 +48,25 @@ def test_iast_leaks(): half_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 current_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 - if args.graph: - rss_list.append(int(current_rss)) - if i % args.printEvery == 0: + if i % print_every == 0: print(f"Round {i} Max RSS: {current_rss}") final_rss = current_rss - print(f"Round {args.iterations} Max RSS: {final_rss}") - - if args.graph: - # TODO: write an ascii graph - pass + print(f"Round {iterations} Max RSS: {final_rss}") percent_increase = ((final_rss - half_rss) / half_rss) * 100 - if percent_increase > args.failPercent: + if percent_increase > fail_percent: print( - f"Failed: memory increase from half-point ({half_iterations} iterations) is {percent_increase:.2f}% which is greater than {args.failPercent}%" + f"Failed: memory increase from half-point ({half_iterations} iterations) is " + "{percent_increase:.2f}% which is greater than {fail_percent}%" ) return 1 else: print( - f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} iterations) which is less than {args.failPercent}%" + f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} " + "iterations) which is less than {fail_percent}%" ) return 0 @@ -79,4 +75,6 @@ def test_iast_leaks(): if __name__ == "__main__": - sys.exit(test_iast_leaks()) + args = parse_arguments() + with override_env({"DD_IAST_ENABLED": "True"}): + sys.exit(test_iast_leaks(args.iterations, args.fail_percent, args.print_every)) diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index aa154588be1..45febbb96ee 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -191,3 +191,10 @@ def test_stacktrace_memory_random_string_check(): file_name, line_number = frame_info assert file_name == "" assert line_number == -1 + + +def test_aggregated_leaks(): + with override_env({"DD_IAST_ENABLED": "True"}): + from scripts.iast.test_leak_functions import test_iast_leaks + + assert test_iast_leaks(100000, 2.0, 500) == 0 From 2513915b691cc027f2f47217aec7620bf8c41d22 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 30 Aug 2024 18:55:07 +0200 Subject: [PATCH 03/14] remove unneded else Signed-off-by: Juanjo Alvarez --- scripts/iast/test_leak_functions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/iast/test_leak_functions.py b/scripts/iast/test_leak_functions.py index f1b94c90f9a..4589cba7c20 100644 --- a/scripts/iast/test_leak_functions.py +++ b/scripts/iast/test_leak_functions.py @@ -63,12 +63,12 @@ def test_iast_leaks(iterations: int, fail_percent: float, print_every: int): "{percent_increase:.2f}% which is greater than {fail_percent}%" ) return 1 - else: - print( - f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} " - "iterations) which is less than {fail_percent}%" - ) - return 0 + + print( + f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} " + "iterations) which is less than {fail_percent}%" + ) + return 0 except KeyboardInterrupt: print("Test interrupted.") From f0cc2343427c5a3ffad904728fb3710cae0dae23 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 2 Sep 2024 10:21:45 +0200 Subject: [PATCH 04/14] Pass -s to pycheck so it doesnt capture the output and publish output more frequently Signed-off-by: Juanjo Alvarez --- riotfile.py | 2 +- tests/appsec/iast_memcheck/test_iast_mem_check.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/riotfile.py b/riotfile.py index 3c6e02a1db3..207feabfcab 100644 --- a/riotfile.py +++ b/riotfile.py @@ -157,7 +157,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): Venv( name="appsec_iast_memcheck", pys=select_pys(min_version="3.8"), - command="pytest {cmdargs} --memray --stacks=35 tests/appsec/iast_memcheck/", + command="pytest {cmdargs} -s --memray --stacks=35 tests/appsec/iast_memcheck/", pkgs={ "requests": latest, "pycryptodome": latest, diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index 45febbb96ee..598cbb32f76 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -197,4 +197,4 @@ def test_aggregated_leaks(): with override_env({"DD_IAST_ENABLED": "True"}): from scripts.iast.test_leak_functions import test_iast_leaks - assert test_iast_leaks(100000, 2.0, 500) == 0 + assert test_iast_leaks(100000, 2.0, 100) == 0 From a17e91d99c19efeb072a06c361eefa24a1c08491 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 2 Sep 2024 11:00:59 +0200 Subject: [PATCH 05/14] fix Signed-off-by: Juanjo Alvarez --- tests/appsec/iast_memcheck/test_iast_mem_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index 598cbb32f76..0a790e7f921 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -193,6 +193,7 @@ def test_stacktrace_memory_random_string_check(): assert line_number == -1 +@pytest.mark.skip(reason="Hangs on CI around 3500 iterations") def test_aggregated_leaks(): with override_env({"DD_IAST_ENABLED": "True"}): from scripts.iast.test_leak_functions import test_iast_leaks From 59dc09229f8673f108906b753988218d37e0e2e1 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 2 Sep 2024 11:01:19 +0200 Subject: [PATCH 06/14] fix Signed-off-by: Juanjo Alvarez --- tests/appsec/iast_memcheck/test_iast_mem_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index 0a790e7f921..598cbb32f76 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -193,7 +193,6 @@ def test_stacktrace_memory_random_string_check(): assert line_number == -1 -@pytest.mark.skip(reason="Hangs on CI around 3500 iterations") def test_aggregated_leaks(): with override_env({"DD_IAST_ENABLED": "True"}): from scripts.iast.test_leak_functions import test_iast_leaks From bafe0743615c1fc225249e462a6143b51646b225 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 2 Sep 2024 15:04:22 +0200 Subject: [PATCH 07/14] Fix fstring and increase iterations Signed-off-by: Juanjo Alvarez --- scripts/iast/test_leak_functions.py | 4 ++-- tests/appsec/iast_memcheck/test_iast_mem_check.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/iast/test_leak_functions.py b/scripts/iast/test_leak_functions.py index 4589cba7c20..bb6f0a28bda 100644 --- a/scripts/iast/test_leak_functions.py +++ b/scripts/iast/test_leak_functions.py @@ -60,13 +60,13 @@ def test_iast_leaks(iterations: int, fail_percent: float, print_every: int): if percent_increase > fail_percent: print( f"Failed: memory increase from half-point ({half_iterations} iterations) is " - "{percent_increase:.2f}% which is greater than {fail_percent}%" + f"{percent_increase:.2f}% which is greater than {fail_percent}%" ) return 1 print( f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} " - "iterations) which is less than {fail_percent}%" + f"iterations) which is less than {fail_percent}%" ) return 0 diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index 598cbb32f76..9d972ddfeb2 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -197,4 +197,4 @@ def test_aggregated_leaks(): with override_env({"DD_IAST_ENABLED": "True"}): from scripts.iast.test_leak_functions import test_iast_leaks - assert test_iast_leaks(100000, 2.0, 100) == 0 + assert test_iast_leaks(100000, 2.0, 1000) == 0 From 951866eccd8e53e7595b4f8981ed51f686e3b886 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 2 Sep 2024 15:14:55 +0200 Subject: [PATCH 08/14] lower iterations Signed-off-by: Juanjo Alvarez --- tests/appsec/iast_memcheck/test_iast_mem_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index 9d972ddfeb2..9d8189c0e75 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -197,4 +197,4 @@ def test_aggregated_leaks(): with override_env({"DD_IAST_ENABLED": "True"}): from scripts.iast.test_leak_functions import test_iast_leaks - assert test_iast_leaks(100000, 2.0, 1000) == 0 + assert test_iast_leaks(100000, 2.0, 50) == 0 From f7403e16337e18adc87488dc22b24570d26cd54b Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 13 Sep 2024 11:26:09 +0200 Subject: [PATCH 09/14] Move the test to a different dir and run them with gitlab Signed-off-by: Juanjo Alvarez --- .gitlab/tests/appsec.yml | 7 ++++++ hatch.toml | 23 +++++++++++++++++++ riotfile.py | 2 +- .../iast_aggregated_memcheck/__init__.py | 0 .../test_aggregated_memleaks.py | 8 +++++++ .../iast_memcheck/test_iast_mem_check.py | 7 ------ 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 tests/appsec/iast_aggregated_memcheck/__init__.py create mode 100644 tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py diff --git a/.gitlab/tests/appsec.yml b/.gitlab/tests/appsec.yml index fb17a41b7f5..dc27966999e 100644 --- a/.gitlab/tests/appsec.yml +++ b/.gitlab/tests/appsec.yml @@ -51,3 +51,10 @@ appsec threats fastapi: variables: SUITE_NAME: "appsec_threats_fastapi" retry: 2 + +appsec aggregated leak testing: + extends: .test_base_hatch + parallel: 6 + variables: + SUITE_NAME: "appsec_aggregated_leak_testing" + retry: 2 diff --git a/hatch.toml b/hatch.toml index 16b90392797..180df1b5896 100644 --- a/hatch.toml +++ b/hatch.toml @@ -289,6 +289,29 @@ test = [ "DD_IAST_ENABLED=true DD_IAST_REQUEST_SAMPLING=100 python -m pytest tests/appsec/contrib_appsec/test_fastapi.py" ] +# ASM Appsec Aggregated Leak Testing + +[envs.appsec_aggregated_leak_testing] +template = "appsec_aggregated_leak_testing" +dependencies = [ + "pytest", + "pytest-cov", +] + +[envs.appsec_aggregated_leak_testing.env-vars] +CMAKE_BUILD_PARALLEL_LEVEL = "12" + +[envs.appsec_aggregated_leak_testing.scripts] +test = [ + "uname -a", + "pip freeze", + "python -m pytest tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py", +] + +[[envs.appsec_aggregated_leak_testing.matrix]] +python = ["3.7", "3.9", "3.10", "3.11", "3.12"] + + # if you add or remove a version here, please also update the parallelism parameter # in .circleci/config.templ.yml [[envs.appsec_threats_fastapi.matrix]] diff --git a/riotfile.py b/riotfile.py index 97fd94c00dd..e6ad0b23546 100644 --- a/riotfile.py +++ b/riotfile.py @@ -157,7 +157,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): Venv( name="appsec_iast_memcheck", pys=select_pys(min_version="3.8"), - command="pytest {cmdargs} -s --memray --stacks=35 tests/appsec/iast_memcheck/", + command="pytest {cmdargs} --memray --stacks=35 tests/appsec/iast_memcheck/", pkgs={ "requests": latest, "pycryptodome": latest, diff --git a/tests/appsec/iast_aggregated_memcheck/__init__.py b/tests/appsec/iast_aggregated_memcheck/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py b/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py new file mode 100644 index 00000000000..a31ce3400ea --- /dev/null +++ b/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py @@ -0,0 +1,8 @@ +from tests.utils import override_env + + +def test_aggregated_leaks(): + with override_env({"DD_IAST_ENABLED": "True"}): + from scripts.iast.test_leak_functions import test_iast_leaks + + assert test_iast_leaks(100000, 2.0, 100) == 0 diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index 9d8189c0e75..aa154588be1 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -191,10 +191,3 @@ def test_stacktrace_memory_random_string_check(): file_name, line_number = frame_info assert file_name == "" assert line_number == -1 - - -def test_aggregated_leaks(): - with override_env({"DD_IAST_ENABLED": "True"}): - from scripts.iast.test_leak_functions import test_iast_leaks - - assert test_iast_leaks(100000, 2.0, 50) == 0 From 1b2ecc56b0744b77ea0627fa350912d818b3809b Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 13 Sep 2024 11:28:35 +0200 Subject: [PATCH 10/14] I cant count Signed-off-by: Juanjo Alvarez --- hatch.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch.toml b/hatch.toml index 180df1b5896..1547480cf79 100644 --- a/hatch.toml +++ b/hatch.toml @@ -309,7 +309,7 @@ test = [ ] [[envs.appsec_aggregated_leak_testing.matrix]] -python = ["3.7", "3.9", "3.10", "3.11", "3.12"] +python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] # if you add or remove a version here, please also update the parallelism parameter From 46d9e7760f22d23a4d396cc777028a7e2a3f4e21 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 13 Sep 2024 11:32:29 +0200 Subject: [PATCH 11/14] Add hypotesis to the aggregated leak testing Signed-off-by: Juanjo Alvarez --- hatch.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/hatch.toml b/hatch.toml index 1547480cf79..65d9865b329 100644 --- a/hatch.toml +++ b/hatch.toml @@ -296,6 +296,7 @@ template = "appsec_aggregated_leak_testing" dependencies = [ "pytest", "pytest-cov", + "hypothesis", ] [envs.appsec_aggregated_leak_testing.env-vars] From aaea73526250ee276b9ee28f932c555d88df2680 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 13 Sep 2024 11:36:05 +0200 Subject: [PATCH 12/14] another dependency Signed-off-by: Juanjo Alvarez --- hatch.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/hatch.toml b/hatch.toml index 65d9865b329..6ae7f0e1a76 100644 --- a/hatch.toml +++ b/hatch.toml @@ -297,6 +297,7 @@ dependencies = [ "pytest", "pytest-cov", "hypothesis", + "requests", ] [envs.appsec_aggregated_leak_testing.env-vars] From f1b369bc97f38657fb59d408529a43fa26993ffe Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 13 Sep 2024 11:43:24 +0200 Subject: [PATCH 13/14] fix conflict Signed-off-by: Juanjo Alvarez --- .gitlab/tests/appsec.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/tests/appsec.yml b/.gitlab/tests/appsec.yml index 1561bcbf68e..85d8e833888 100644 --- a/.gitlab/tests/appsec.yml +++ b/.gitlab/tests/appsec.yml @@ -52,17 +52,16 @@ appsec threats fastapi: SUITE_NAME: "appsec_threats_fastapi" retry: 2 -<<<<<<< juanjux/better-leak-script-and-testing appsec aggregated leak testing: extends: .test_base_hatch parallel: 6 variables: SUITE_NAME: "appsec_aggregated_leak_testing" -======= + retry: 2 + appsec iast native: extends: .test_base_hatch parallel: 6 variables: SUITE_NAME: "appsec_iast_native" ->>>>>>> main retry: 2 From a307f586c23b2933bc28ca04427ef7a34a346da2 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 13 Sep 2024 12:06:33 +0200 Subject: [PATCH 14/14] Increase memcheck timeout Signed-off-by: Juanjo Alvarez --- .gitlab/tests/appsec.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/tests/appsec.yml b/.gitlab/tests/appsec.yml index 85d8e833888..b4fd757bf63 100644 --- a/.gitlab/tests/appsec.yml +++ b/.gitlab/tests/appsec.yml @@ -58,6 +58,7 @@ appsec aggregated leak testing: variables: SUITE_NAME: "appsec_aggregated_leak_testing" retry: 2 + timeout: 25m appsec iast native: extends: .test_base_hatch