From 04b6c72b4017f5ea24060267c0b0ca3cf2ccd72b Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 14 Jun 2023 10:48:04 +0200 Subject: [PATCH 01/18] testing ct hook for GA reporting for testing use the following commands: . tools/test-runner-complete.sh test-runner.sh --skip-preset --skip-build-mim --skip-small-tests --skip-start-nodes --skip-stop-nodes --db --skip-cover --skip-preset --dev-nodes --spec failure_reporting_testing.spec --- big_tests/failure_reporting_testing.spec | 12 +++ big_tests/src/ct_test_hook.erl | 101 +++++++++++++++++++++++ big_tests/tests/test_SUITE.erl | 86 +++++++++++++++++++ tools/test.sh | 10 ++- 4 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 big_tests/failure_reporting_testing.spec create mode 100644 big_tests/src/ct_test_hook.erl create mode 100644 big_tests/tests/test_SUITE.erl diff --git a/big_tests/failure_reporting_testing.spec b/big_tests/failure_reporting_testing.spec new file mode 100644 index 0000000000..57a486187f --- /dev/null +++ b/big_tests/failure_reporting_testing.spec @@ -0,0 +1,12 @@ +{suites, "tests", test_SUITE}. + +{config, ["test.config"]}. +{logdir, "ct_report"}. + +%% ct_test_hook should be executed just once, +%% see ct_test_hook:id/1 implementation and +%% ct_hooks documentation: +%% https://erlang.org/doc/man/ct_hooks.html +{ct_hooks, [ct_test_hook, ct_test_hook]}. + +% {ct_hooks, [ct_groups_summary_hook, ct_markdown_errors_hook]}. diff --git a/big_tests/src/ct_test_hook.erl b/big_tests/src/ct_test_hook.erl new file mode 100644 index 0000000000..74e2438165 --- /dev/null +++ b/big_tests/src/ct_test_hook.erl @@ -0,0 +1,101 @@ +-module(ct_test_hook). +-export([id/1]). %% required to prevent multiple hook executions +-export([init/2, terminate/1]). +-export([post_all/3, post_groups/2]). +-export([on_tc_fail/4, on_tc_skip/4]). +-export([pre_init_per_suite/3, post_init_per_suite/4, + pre_end_per_suite/3, post_end_per_suite/4]). +-export([pre_init_per_group/4, post_init_per_group/5, + pre_end_per_group/4, post_end_per_group/5]). +-export([pre_init_per_testcase/4, post_init_per_testcase/5, + pre_end_per_testcase/4, post_end_per_testcase/5]). + +-record(state,{log = [], ref = make_ref()}). +-define(MFA, {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}). +% -define(ADD_LOG(State), (State)). +-define(ADD_LOG(State), (State#state{log = [?MFA | State#state.log]})). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% CT hook callbacks +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +id(_Opts)-> + start_tracing(), + ?MODULE. + +init(_Id, _Opts) -> + start_tracing(), + {ok, ?ADD_LOG(#state{})}. + +terminate(_CTHState) -> + stop_tracing(). + +on_tc_fail(_SuiteName, _TestName, _Reason, CTHState) -> + ?ADD_LOG(CTHState). + +on_tc_skip(_SuiteName, _TestName, _Reason, CTHState) -> + ?ADD_LOG(CTHState). + +post_all(_SuiteName, Return, _GroupDefs) -> Return. + +post_groups(_SuiteName, GroupDefs) -> GroupDefs. + +pre_init_per_suite(_SuiteName, InitData, CTHState) when is_list(InitData)-> + {[{ct_test_hook, true} | InitData], ?ADD_LOG(CTHState)}; +pre_init_per_suite(_SuiteName, InitData, CTHState) -> + {InitData, ?ADD_LOG(CTHState)}. + +post_init_per_suite(_SuiteName, _Config, Return, CTHState) -> + {Return, ?ADD_LOG(CTHState)}. + +pre_end_per_suite(_SuiteName, EndData, CTHState) -> + {EndData, ?ADD_LOG(CTHState)}. + +post_end_per_suite(_SuiteName, _Config, Return, CTHState) -> + {Return, ?ADD_LOG(CTHState)}. + +pre_init_per_group(_SuiteName, _GroupName, InitData, CTHState) -> + {InitData, ?ADD_LOG(CTHState)}. + +post_init_per_group(_SuiteName, _GroupName, _Config, Return, CTHState) -> + {Return, ?ADD_LOG(CTHState)}. + +pre_end_per_group(_SuiteName, _GroupName, EndData, CTHState) -> + {EndData, ?ADD_LOG(CTHState)}. + +post_end_per_group(_SuiteName, _GroupName, _Config, Return, CTHState) -> + {Return, ?ADD_LOG(CTHState)}. + +pre_init_per_testcase(_SuiteName, _TestcaseName, InitData, CTHState) -> + {InitData, ?ADD_LOG(CTHState)}. + +post_init_per_testcase(_SuiteName, _TestcaseName, _Config, Return, CTHState) -> + {Return, ?ADD_LOG(CTHState)}. + +pre_end_per_testcase(_SuiteName, _TestcaseName, EndData, CTHState) -> + {EndData, ?ADD_LOG(CTHState)}. + +post_end_per_testcase(_SuiteName, _TestcaseName, _Config, Return, CTHState) -> + {Return, ?ADD_LOG(CTHState)}. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% local functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +start_tracing() -> + %% this function is idempotent and can be safely called multiple times + case dbg:tracer(process, {fun tracer_fun/2, ok}) of + {ok, _Pid} -> + dbg:p(all, call), + dbg:tp(?MODULE, []); + {error, already_started} -> + ok + end. + +tracer_fun(Msg, State) -> + io:format("~n!DBG!: ~p~n", [Msg]), + State. + +stop_tracing() -> + %% this function is idempotent and can be safely called multiple times + timer:sleep(10000), + dbg:stop_clear(). diff --git a/big_tests/tests/test_SUITE.erl b/big_tests/tests/test_SUITE.erl new file mode 100644 index 0000000000..4118fe6399 --- /dev/null +++ b/big_tests/tests/test_SUITE.erl @@ -0,0 +1,86 @@ +-module(test_SUITE). + +-compile([export_all, nowarn_export_all]). + +% -define(ERROR(E), (throw(E))). +-define(ERROR(E), (error(E))). +% -define(ERROR(E), (exit(E))). + +all() -> + [{group, test_group}, + {group, skipped_test_group}, + {group, failing_test_group_1}, + {group, failing_test_group_2}, + {group, nested_test_group}, + {group, deeply_nested_test_group}, + {group, failing_nested_test_group_1}, + {group, failing_nested_test_group_2} + | test_cases() ]. + +groups() -> + [{test_group, [], test_cases()}, + {skipped_test_group, [], test_cases()}, + {failing_test_group_1, [], test_cases()}, + {failing_test_group_2, [], test_cases()}, + {nested_test_group, [], [{group, test_group}, + {group, skipped_test_group}, + {group, failing_test_group_1}, + {group, failing_test_group_2}]}, + {deeply_nested_test_group, [], [{group, test_group}, + {group, skipped_test_group}, + {group, failing_test_group_1}, + {group, failing_test_group_2}, + {group, nested_test_group}]}, + {failing_nested_test_group_1, [], [{group, test_group}]}, + {failing_nested_test_group_2, [], [{group, test_group}]}]. + +test_cases() -> + [passing_tc, + skipped_tc, + failing_tc_1, + failing_tc_2, + failing_tc_3]. + +% init_per_suite(_Config) -> ?ERROR({error, init_per_suite}); +init_per_suite(Config) -> Config. + +% end_per_suite(_Config) -> ?ERROR({error, end_per_suite}); +end_per_suite(Config) -> Config. + +init_per_group(skipped_test_group, Config) -> + {skip, init_per_group}; +init_per_group(failing_test_group_1, Config) -> + ?ERROR({error, init_per_group}); +init_per_group(failing_nested_test_group_1, Config) -> + ?ERROR({error, init_per_group}); +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(failing_test_group_2, Config) -> + ?ERROR({error, end_per_group}); +end_per_group(failing_nested_test_group_2, Config) -> + ?ERROR({error, end_per_group}); +end_per_group(_GroupName, Config) -> + Config. + +init_per_testcase(skipped_tc, Config) -> + {skip, init_per_testcase}; +init_per_testcase(failing_tc_2, Config) -> + ?ERROR({error, init_per_testcase}); +init_per_testcase(_CaseName, Config) -> + Config. + +end_per_testcase(failing_tc_3, Config) -> + ?ERROR({error, end_per_testcase}); +end_per_testcase(_CaseName, Config) -> + Config. + +passing_tc(Config) -> ok. + +skipped_tc(Config) -> ok. + +failing_tc_1(Config) -> ?ERROR({error, testcase}). + +failing_tc_2(Config) -> ok. + +failing_tc_3(Config) -> ok. diff --git a/tools/test.sh b/tools/test.sh index a9ddcb2463..4569a0cf5f 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -92,8 +92,8 @@ run_small_tests() { export REBAR_CT_EXTRA_ARGS="$REBAR_CT_EXTRA_ARGS" make ct tools/print-dots.sh stop - SMALL_SUMMARIES_DIRS=${BASE}/_build/test/logs/ct_run* - SMALL_SUMMARIES_DIR=$(choose_newest_directory ${SMALL_SUMMARIES_DIRS}) + SMALL_SUMMARIES_DIRS=( ${BASE}/_build/test/logs/ct_run* ) + SMALL_SUMMARIES_DIR=$(choose_newest_directory "${SMALL_SUMMARIES_DIRS[@]}") ${TOOLS}/summarise-ct-results ${SMALL_SUMMARIES_DIR} } @@ -163,7 +163,8 @@ run_test_preset() { print_running_nodes() { echo "Running nodes:" # Expand wildcard into a bash array - EPMDS=( "${BASE}"/_build/mim1/rel/mongooseim/erts-*/bin/epmd ) + # if there's no mim1 build, fallback to epmd + EPMDS=( "${BASE}"/_build/mim1/rel/mongooseim/erts-*/bin/epmd epmd ) # Missing index expands into ${EPMDS[0]} "$EPMDS" -names } @@ -228,7 +229,8 @@ run_tests() { echo echo "All tests done." - grep "fail_ci_build=true" ${BASE}/_build/mim*/rel/mongooseim/log/mongooseim.log.1 + local log_files=( $( compgen -G "${BASE}/_build/mim"*"/rel/mongooseim/log/mongooseim.log.1" ) ) + [ 0 -ne "${#log_files[@]}" ] && grep -F "fail_ci_build=true" "${log_files[@]}" # If phrase found than exit with code 1 test $? -eq 1 LOG_STATUS=$? From b08cda8d39f5a588b24246fead9799d7905188a2 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Thu, 15 Jun 2023 10:25:59 +0200 Subject: [PATCH 02/18] updating ct_markdown_errors_hook reporting tastcase as failed if init_per_testcase or end_per_testcase function crashes check /tmp/ct_summary file after test execution: cat /tmp/ct_summary for testing use the following commands: . tools/test-runner-complete.sh test-runner.sh --skip-preset --skip-build-mim --skip-small-tests --skip-start-nodes --skip-stop-nodes --db --skip-cover --skip-preset --dev-nodes --verbose --spec failure_reporting_testing.spec --- big_tests/failure_reporting_testing.spec | 2 +- big_tests/src/ct_markdown_errors_hook.erl | 30 +++++++++++++---------- big_tests/src/ct_test_hook.erl | 4 ++- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/big_tests/failure_reporting_testing.spec b/big_tests/failure_reporting_testing.spec index 57a486187f..65d13c3313 100644 --- a/big_tests/failure_reporting_testing.spec +++ b/big_tests/failure_reporting_testing.spec @@ -9,4 +9,4 @@ %% https://erlang.org/doc/man/ct_hooks.html {ct_hooks, [ct_test_hook, ct_test_hook]}. -% {ct_hooks, [ct_groups_summary_hook, ct_markdown_errors_hook]}. +{ct_hooks, [ct_groups_summary_hook, ct_markdown_errors_hook]}. diff --git a/big_tests/src/ct_markdown_errors_hook.erl b/big_tests/src/ct_markdown_errors_hook.erl index 01f72e9d7e..515b229f79 100644 --- a/big_tests/src/ct_markdown_errors_hook.erl +++ b/big_tests/src/ct_markdown_errors_hook.erl @@ -8,12 +8,12 @@ -export([id/1]). -export([init/2]). -export([post_init_per_suite/4, - post_init_per_group/4, - post_init_per_testcase/4]). + post_init_per_group/5, + post_init_per_testcase/5]). -export([post_end_per_suite/4, - post_end_per_group/4, - post_end_per_testcase/4]). --record(state, { file, summary_file, truncated_counter_file, suite, limit }). + post_end_per_group/5, + post_end_per_testcase/5]). +-record(state, { file, summary_file, truncated_counter_file, limit }). %% @doc Return a unique id for this CTH. id(_Opts) -> @@ -32,26 +32,26 @@ init(_Id, _Opts) -> post_init_per_suite(SuiteName, Config, Return, State) -> State2 = handle_return(SuiteName, init_per_suite, Return, Config, State), - {Return, State2#state{suite = SuiteName}}. + {Return, State2}. -post_init_per_group(_GroupName, Config, Return, State=#state{suite = SuiteName}) -> +post_init_per_group(SuiteName, _GroupName, Config, Return, State) -> State2 = handle_return(SuiteName, init_per_group, Return, Config, State), - {Return, State2#state{}}. + {Return, State2}. -post_init_per_testcase(TC, Config, Return, State=#state{suite = SuiteName}) -> +post_init_per_testcase(SuiteName, TC, Config, Return, State) -> State2 = handle_return(SuiteName, TC, Return, Config, State), {Return, State2}. post_end_per_suite(SuiteName, Config, Return, State) -> State2 = handle_return(SuiteName, end_per_suite, Return, Config, State), - {Return, State2#state{suite = ''}}. + {Return, State2}. -post_end_per_group(_GroupName, Config, Return, State=#state{suite = SuiteName}) -> +post_end_per_group(SuiteName, _GroupName, Config, Return, State) -> State2 = handle_return(SuiteName, end_per_group, Return, Config, State), - {Return, State2#state{}}. + {Return, State2}. %% @doc Called after each test case. -post_end_per_testcase(TC, Config, Return, State=#state{suite = SuiteName}) -> +post_end_per_testcase(SuiteName, TC, Config, Return, State) -> State2 = handle_return(SuiteName, TC, Return, Config, State), {Return, State2}. @@ -145,8 +145,12 @@ to_error_message(Return) -> Return; {fail, _} -> Return; + {failed, _} -> %% a special case for the crash in end_per_testcase + Return; {error, _} -> Return; + {skip, {failed, _}} -> %% a special case for the crash in init_per_testcase + Return; {skip, _} -> ok; _ -> diff --git a/big_tests/src/ct_test_hook.erl b/big_tests/src/ct_test_hook.erl index 74e2438165..2df1017d70 100644 --- a/big_tests/src/ct_test_hook.erl +++ b/big_tests/src/ct_test_hook.erl @@ -86,7 +86,9 @@ start_tracing() -> case dbg:tracer(process, {fun tracer_fun/2, ok}) of {ok, _Pid} -> dbg:p(all, call), - dbg:tp(?MODULE, []); + % dbg:tp(?MODULE, []), + dbg:tpl(ct_markdown_errors_hook, cx), + ok; {error, already_started} -> ok end. From db22807d4b4db197b087350fe42eaba5f790496a Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Thu, 15 Jun 2023 13:04:42 +0200 Subject: [PATCH 03/18] adding gh-report-failing-testcases-to-ga4.sh script --- tools/gh-report-failing-testcases-to-ga4.sh | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 tools/gh-report-failing-testcases-to-ga4.sh diff --git a/tools/gh-report-failing-testcases-to-ga4.sh b/tools/gh-report-failing-testcases-to-ga4.sh new file mode 100755 index 0000000000..9856dd905a --- /dev/null +++ b/tools/gh-report-failing-testcases-to-ga4.sh @@ -0,0 +1,31 @@ +api_secret="$GA4_API_SECRET" +measurement_id="$GA4_MEASUREMENT_API" + +ct_summary_file="/tmp/ct_summary" + +echo "gh-report-failing-testcases-to-ga4.sh" + +[ -f "$ct_summary_file" ] || { echo "ct_summary file is missing"; exit 1; } +[ -z "$api_secret" ] && { echo "invalid \$GA4_API_SECRET value"; exit 1; } +[ -z "$measurement_id" ] && { echo "invalid \$GA4_MEASUREMENT_API value"; exit 1; } + +url="https://www.google-analytics.com/mp/collect" +url+="?measurement_id=${measurement_id}" +url+="&api_secret=${api_secret}" + +function report_tc_to_ga4() { + local test_case="$1" + local client_id="$RANDOM" + local payload='{"client_id": "'"$client_id"'", + "events": [{ + "name": "failed_test_case", + "params": { + "test_case": "'"$test_case"'"}}]}' + + echo "reporting failed test case '${test_case}'" + curl -s -X POST "$url" -d "$payload" +} + +while read tc; do + report_tc_to_ga4 "$tc" +done < "$ct_summary_file" From 480079a623ab40d6954a5cc52f5dd4ff69b342c3 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Sun, 18 Jun 2023 00:49:50 +0200 Subject: [PATCH 04/18] adding big_tests reporting to GA4 for GH actions --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 680700d60d..6e2f063665 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,9 @@ env: AWS_DEFAULT_REGION: "${{ secrets.AWS_DEFAULT_REGION }}" AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # required for tools/gh-report-failing-testcases-to-ga4.sh script + GA4_API_SECRET: "${{ secrets.GA4_API_SECRET }}" + GA4_MEASUREMENT_API: "${{ secrets.GA4_MEASUREMENT_API }}" jobs: small_tests: @@ -80,6 +83,9 @@ jobs: - name: upload common test results on failure if: ${{ failure() }} run: tools/gh-upload-to-s3.sh big_tests/ct_report + - name: upload big_tests results to GA4 + if: ${{ always() }} + run: tools/gh-report-failing-testcases-to-ga4.sh dynamic_domains_big_tests: name: dynamic domains ${{matrix.preset}} on OTP ${{matrix.otp}} @@ -105,6 +111,9 @@ jobs: - name: upload common test results on failure if: ${{ failure() }} run: tools/gh-upload-to-s3.sh big_tests/ct_report + - name: upload big_tests results to GA4 + if: ${{ always() }} + run: tools/gh-report-failing-testcases-to-ga4.sh coveralls_webhook: needs: [big_tests, small_tests, dynamic_domains_big_tests] From c003872ab747dad69b339bc7f7f18be440f86c0d Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Tue, 27 Jun 2023 00:34:44 +0200 Subject: [PATCH 05/18] adding big_tests reporting to GA4 for CircleCI --- .circleci/template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/template.yml b/.circleci/template.yml index 1e47e1557e..9157d4c26a 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -302,6 +302,13 @@ commands: command: | tools/circleci-prepare-log-dir.sh if [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then tools/circleci-upload-to-s3.sh; fi + report_failed_test_cases_to_ga4: + steps: + - run: + name: Report failed test cases + when: always + command: | + tools/gh-report-failing-testcases-to-ga4.sh publish_github_comment: steps: - run: @@ -630,6 +637,7 @@ jobs: tail -100 _build/mim1/rel/mongooseim/log/mongooseim.log.1 - upload_results_to_aws - publish_github_comment + - report_failed_test_cases_to_ga4 dialyzer: executor: << parameters.executor >> From 3dec5c12dec3ac5a8f1938fd51ab07e61675c017 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Tue, 27 Jun 2023 05:06:19 +0200 Subject: [PATCH 06/18] use mam.spec for elasticsearch_and_cassandra_25 CircleCI job --- .circleci/template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/template.yml b/.circleci/template.yml index 9157d4c26a..9bc226581a 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -951,6 +951,7 @@ workflows: filters: *all_tags - big_tests_in_docker: name: elasticsearch_and_cassandra_25 + spec: mam.spec executor: otp_25_elasticsearch_cassandra context: mongooseim-org preset: elasticsearch_and_cassandra_mnesia From 3d8b6b270d3cb3292cfdf7a8998b4399e50f805d Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 28 Jun 2023 10:21:28 +0200 Subject: [PATCH 07/18] adding description for ct_mongoose_log_hook --- big_tests/src/ct_mongoose_log_hook.erl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/big_tests/src/ct_mongoose_log_hook.erl b/big_tests/src/ct_mongoose_log_hook.erl index 14706b860d..3776450abd 100644 --- a/big_tests/src/ct_mongoose_log_hook.erl +++ b/big_tests/src/ct_mongoose_log_hook.erl @@ -1,4 +1,7 @@ -%%% @doc Copy mongooseim.log into CT reports +%%% @doc This hook copies mongooseim.log into an html file (with labels), +%%% and inserts links into test case specific html report files. note that +%%% a temporary html file is created for each link in log_private directory, +%%% for more information see comments for add_log_link_to_line/5 function. -module(ct_mongoose_log_hook). %% @doc Add the following line in your *.spec file to @@ -220,13 +223,6 @@ post_insert_line_numbers_into_report(State=#state{node=Node, reader=Reader, writ file:write(Writer, Message), State#state{current_line_num=CurrentLineNum2}. -add_log_link_to_line(PrivDir, UrlFile, LogLine, Node, ExtraDescription) -> - Label = "L" ++ integer_to_list(LogLine), - Heading = "View log from node " ++ atom_to_list(Node) ++ ExtraDescription, - %% We need to invent something unique enough here :) - LinkName = atom_to_list(Node) ++ "_" ++ integer_to_list(LogLine) ++ ".html", - add_log_link(Heading, PrivDir, LinkName, UrlFile, Label). - %% Function `escalus_ct:add_log_link(Heading, URL, Type).' %% allows to add simple links. %% @@ -240,7 +236,11 @@ add_log_link_to_line(PrivDir, UrlFile, LogLine, Node, ExtraDescription) -> %% `LinkName' - filename where to write our redirect code inside log_private %% `UrlFile' - destination URL to redirect to %% `Label' - position in the document -add_log_link(Heading, PrivDir, LinkName, UrlFile, Label) -> +add_log_link_to_line(PrivDir, UrlFile, LogLine, Node, ExtraDescription) -> + Label = "L" ++ integer_to_list(LogLine), + Heading = "View log from node " ++ atom_to_list(Node) ++ ExtraDescription, + %% We need to invent something unique enough here :) + LinkName = atom_to_list(Node) ++ "_" ++ integer_to_list(LogLine) ++ ".html", URL = UrlFile ++ "#" ++ Label, RedirectCode = "", WhereToWrite = filename:join(PrivDir, LinkName), From b6c35a4886e1ab8e7760c40f00ad4fd11387ebc4 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Tue, 4 Jul 2023 23:29:22 +0200 Subject: [PATCH 08/18] improving ct_groups_summary_hook --- big_tests/failure_reporting_testing.spec | 29 ++++++-- big_tests/run_common_test.erl | 6 ++ big_tests/src/ct_groups_summary_hook.erl | 89 +++++++++++++++--------- big_tests/src/ct_test_hook.erl | 3 +- big_tests/tests/test_SUITE.erl | 70 ++++++++++++------- tools/summarise-ct-results | 8 ++- 6 files changed, 139 insertions(+), 66 deletions(-) diff --git a/big_tests/failure_reporting_testing.spec b/big_tests/failure_reporting_testing.spec index 65d13c3313..cc56cf1e5e 100644 --- a/big_tests/failure_reporting_testing.spec +++ b/big_tests/failure_reporting_testing.spec @@ -3,10 +3,29 @@ {config, ["test.config"]}. {logdir, "ct_report"}. -%% ct_test_hook should be executed just once, -%% see ct_test_hook:id/1 implementation and -%% ct_hooks documentation: -%% https://erlang.org/doc/man/ct_hooks.html +%%% ct_test_hook is just a demo CT hook with tracing. +%%% ct_test_hook should be executed just once, see +%%% implementation of ct_test_hook:id/1 and ct_hooks +%%% documentation: +%%% https://erlang.org/doc/man/ct_hooks.html {ct_hooks, [ct_test_hook, ct_test_hook]}. -{ct_hooks, [ct_groups_summary_hook, ct_markdown_errors_hook]}. +%%% this hook generates junit_report.xml file +{ct_hooks, [cth_surefire]}. + +%%% this hook generates groups.summary files per suite +%%% and an overall resulting all_groups.summary file. +{ct_hooks, [ct_groups_summary_hook]}. + +{ct_hooks, [ct_markdown_errors_hook]}. + +%%% this hook prints execution summary in the end of +%%% big_tests run. however, silent_exec.sh script redirects +%%% output into a file. set VERBOSE env variable to 1, to +%%% tail output file. +% {ct_hooks, [ct_tty_hook]}. + +%%% this hook appeands /tmp/progress file, that file +%%% is printed asynchronously by test.sh script using +%%% 'tail -f /tmp/progress &' command. +% {ct_hooks, [ct_progress_hook]}. diff --git a/big_tests/run_common_test.erl b/big_tests/run_common_test.erl index d21372a5e2..cb590e948a 100644 --- a/big_tests/run_common_test.erl +++ b/big_tests/run_common_test.erl @@ -68,6 +68,12 @@ main(RawArgs) -> io:format("Exiting by test cases summary: ~p~n", [ExitStatusByTestCases]), init:stop(ExitStatusByTestCases); _ when is_integer(ExitStatusByGroups) -> + %% FIXME: This is incorrect assumption, it ignores results of all individual + %% tests cases and groups w/o 'repeat_until_all_ok' flag. So we can return + %% false positive result here. It's not too critical, because test.sh script + %% recalculates return code using summarise-ct-results utility. However, it + %% may result in squashing ct output, since execution of run_common_test.erl + %% is wrapped using silent_exec.sh tool. io:format("Exiting by groups summary: ~p~n", [ExitStatusByGroups]), init:stop(ExitStatusByGroups) end diff --git a/big_tests/src/ct_groups_summary_hook.erl b/big_tests/src/ct_groups_summary_hook.erl index 2e1bc8d7cf..76655b7f88 100644 --- a/big_tests/src/ct_groups_summary_hook.erl +++ b/big_tests/src/ct_groups_summary_hook.erl @@ -14,6 +14,10 @@ -include_lib("common_test/include/ct.hrl"). +-record(group_status, {status = ok :: ok | failed, + n_failed = 0 :: pos_integer(), + failed = [] :: list()}). + %% @doc Return a unique id for this CTH. id(_Opts) -> "ct_results_summary_hook_001". @@ -64,41 +68,59 @@ update_group_status(GroupName, Config, State) -> case lists:keyfind(tc_group_properties, 1, Config) of false -> State; {tc_group_properties, Properties} -> - case proplists:is_defined(repeat_until_all_ok, Properties) of - false -> State; - true -> - GroupNameWPath = group_name_with_path(GroupName, Config), - GroupResult = ?config(tc_group_result, Config), - case proplists:get_value(failed, GroupResult, []) of - [] -> - %% If there are no failed cases, then the only skipped cases present - %% in the group result are user-skipped cases. - %% In this case we do not want to fail the whole group. - do_update_group_status(ok, GroupNameWPath, 0, State); - Failed -> - %% If there are failed cases, it doesn't matter if the skipped cases - %% are user-skipped or auto-skipped (which might depend on `sequence` - %% group property being enabled or not) - we fail in either case. - %% TODO: Report to the respective GitHub PR - ct:pal("Failed in this group: ~p", [Failed]), - do_update_group_status(failed, GroupNameWPath, length(Failed), State) - end - end + GroupNameWPath = group_name_with_path(GroupName, Config), + GroupResult = ?config(tc_group_result, Config), + Failed = proplists:get_value(failed, GroupResult, []), + Status = case Failed of + [] -> + %% If there are no failed cases, then the only skipped cases present + %% in the group result are user-skipped cases. + %% In this case we do not want to fail the whole group. + ok; + _ -> + %% If there are failed cases, it doesn't matter if the skipped cases + %% are user-skipped or auto-skipped (which might depend on `sequence` + %% group property being enabled or not) - we fail in either case. + failed + end, + RepeatFlag = proplists:is_defined(repeat_until_all_ok, Properties), + do_update_group_status(RepeatFlag, Status, GroupNameWPath, Failed, State) end. -do_update_group_status(Status, GroupNameWPath, NFailed, #{current_suite := CurrentSuite} = State) -> +do_update_group_status(RepeatFlag, Status, GroupNameWPath, Failed, + #{current_suite := CurrentSuite} = State) -> SuiteState = maps:get(CurrentSuite, State), + NewGroupStatus = #group_status{status = Status, n_failed = length(Failed), + failed = Failed}, NewSuiteState = case maps:get(GroupNameWPath, SuiteState, undefined) of + undefined when RepeatFlag =:= true -> + %% repeat_until_all_ok flag works like a counter. + %% it decreases with every subsequent run, and it + %% is not set at all on the last attempt. so we + %% want add a new group status info only if flag + %% is set. otherwise, we want to update the status + %% only if group is already stored. + SuiteState#{GroupNameWPath => NewGroupStatus}; undefined -> - SuiteState#{GroupNameWPath => group_status(Status, NFailed)}; - {_PrevStatus, {n_failed, NFailedAcc}} -> - SuiteState#{GroupNameWPath => group_status(Status, NFailed + NFailedAcc)} + SuiteState; + PrevGroupStatus -> + SuiteState#{GroupNameWPath := merge_group_status(PrevGroupStatus, + NewGroupStatus)} end, State#{CurrentSuite := NewSuiteState}. -group_status(Status, NFailed) -> - {Status, {n_failed, NFailed}}. +merge_group_status(#group_status{failed = PrevFailed, + n_failed = PrevNFailed} = _PrevGroupStatus, + #group_status{status = NewStatus, failed = NewFailed, + n_failed = NewNFailed} = _NewGroupStatus) -> + MergedFailed = lists:umerge(lists:usort(PrevFailed),lists:usort(NewFailed)), + %% we need to count the number of fails, not the number of failed test cases + %% (i.e. length(MergedFailed)). that's because we compare it with the number + %% of failed test cases reported in suite.summary files, for more details see + %% summarise-ct-results script. + MergedNFailed = PrevNFailed + NewNFailed, + #group_status{status = NewStatus, n_failed = MergedNFailed, failed = MergedFailed}. group_name_with_path(GroupName, Config) -> @@ -131,13 +153,12 @@ write_groups_summary(Config, #{total_ok := TOK, total_failed := TFailed, total_failed := TFailed + Failed, total_eventually_ok_tests := TEvOK + FailedTests}. -acc_groups_summary(_GroupName, {ok, {n_failed, NFailedTests}}, +acc_groups_summary(_GroupName, #group_status{status = ok, n_failed = NFailedTests}, {OkGroupsAcc, FailedGroupsAcc, EventuallyOkAcc}) -> - {OkGroupsAcc + 1, FailedGroupsAcc, - %% The group was repeated, but it eventually passed, so the tests must finally be ok. - EventuallyOkAcc + NFailedTests}; -acc_groups_summary(_GroupName, {__, {n_failed, _NFailedTests}}, + %% Either the group has passed succesfully on the first run, + %% or it was repeated, and eventually passed, so the tests must finally be ok. + {OkGroupsAcc + 1, FailedGroupsAcc, EventuallyOkAcc + NFailedTests}; +acc_groups_summary(_GroupName, #group_status{status = failed, n_failed = NFailedTests}, {OkGroupsAcc, FailedGroupsAcc, EventuallyOkAcc}) -> - {OkGroupsAcc, FailedGroupsAcc + 1, - %% The group never succeeded, the failed tests are NOT eventually ok. - EventuallyOkAcc}. + %% The group never succeeded, the failed tests are NOT eventually ok. + {OkGroupsAcc, FailedGroupsAcc + 1, EventuallyOkAcc}. diff --git a/big_tests/src/ct_test_hook.erl b/big_tests/src/ct_test_hook.erl index 2df1017d70..44a3c70f38 100644 --- a/big_tests/src/ct_test_hook.erl +++ b/big_tests/src/ct_test_hook.erl @@ -87,7 +87,8 @@ start_tracing() -> {ok, _Pid} -> dbg:p(all, call), % dbg:tp(?MODULE, []), - dbg:tpl(ct_markdown_errors_hook, cx), + % dbg:tpl(ct_markdown_errors_hook, cx), + dbg:tp(ct_groups_summary_hook, cx), ok; {error, already_started} -> ok diff --git a/big_tests/tests/test_SUITE.erl b/big_tests/tests/test_SUITE.erl index 4118fe6399..0cc03e145f 100644 --- a/big_tests/tests/test_SUITE.erl +++ b/big_tests/tests/test_SUITE.erl @@ -6,16 +6,19 @@ -define(ERROR(E), (error(E))). % -define(ERROR(E), (exit(E))). -all() -> - [{group, test_group}, - {group, skipped_test_group}, - {group, failing_test_group_1}, - {group, failing_test_group_2}, - {group, nested_test_group}, - {group, deeply_nested_test_group}, - {group, failing_nested_test_group_1}, - {group, failing_nested_test_group_2} - | test_cases() ]. +all() -> [{group, passing_repeating_group_with_autoskip}]. + % [{group, test_group}, + % {group, skipped_test_group}, + % {group, failing_test_group_1}, + % {group, failing_test_group_2}, + % {group, nested_test_group}, + % {group, deeply_nested_test_group}, + % {group, failing_nested_test_group_1}, + % {group, failing_nested_test_group_2}, + % {group, failing_repeating_group}, + % {group, passing_repeating_group}, + % {group, passing_repeating_group_with_autoskip} + % | test_cases() ]. groups() -> [{test_group, [], test_cases()}, @@ -32,7 +35,11 @@ groups() -> {group, failing_test_group_2}, {group, nested_test_group}]}, {failing_nested_test_group_1, [], [{group, test_group}]}, - {failing_nested_test_group_2, [], [{group, test_group}]}]. + {failing_nested_test_group_2, [], [{group, test_group}]}, + {failing_repeating_group, [{repeat_until_all_ok, 3}], test_cases()}, + {passing_repeating_group, [{repeat_until_all_ok, 3}], [passing_tc, eventually_passing_tc]}, + {passing_repeating_group_with_autoskip, [{repeat_until_all_ok, 3}, sequence], [eventually_passing_tc, + passing_tc]}]. test_cases() -> [passing_tc, @@ -42,45 +49,58 @@ test_cases() -> failing_tc_3]. % init_per_suite(_Config) -> ?ERROR({error, init_per_suite}); -init_per_suite(Config) -> Config. +init_per_suite(Config) -> + %% atomics are automatically garbage collected + %% when they are no longer referenced + [ {counter, atomics:new(1, [{signed, false}])} | Config ]. % end_per_suite(_Config) -> ?ERROR({error, end_per_suite}); end_per_suite(Config) -> Config. -init_per_group(skipped_test_group, Config) -> +init_per_group(skipped_test_group, _Config) -> {skip, init_per_group}; -init_per_group(failing_test_group_1, Config) -> +init_per_group(failing_test_group_1, _Config) -> ?ERROR({error, init_per_group}); -init_per_group(failing_nested_test_group_1, Config) -> +init_per_group(failing_nested_test_group_1, _Config) -> ?ERROR({error, init_per_group}); init_per_group(_GroupName, Config) -> Config. -end_per_group(failing_test_group_2, Config) -> +end_per_group(failing_test_group_2, _Config) -> ?ERROR({error, end_per_group}); -end_per_group(failing_nested_test_group_2, Config) -> +end_per_group(failing_nested_test_group_2, _Config) -> ?ERROR({error, end_per_group}); end_per_group(_GroupName, Config) -> Config. -init_per_testcase(skipped_tc, Config) -> +init_per_testcase(skipped_tc, _Config) -> {skip, init_per_testcase}; -init_per_testcase(failing_tc_2, Config) -> +init_per_testcase(failing_tc_2, _Config) -> ?ERROR({error, init_per_testcase}); init_per_testcase(_CaseName, Config) -> Config. -end_per_testcase(failing_tc_3, Config) -> +end_per_testcase(failing_tc_3, _Config) -> ?ERROR({error, end_per_testcase}); end_per_testcase(_CaseName, Config) -> Config. -passing_tc(Config) -> ok. +passing_tc(_Config) -> ok. -skipped_tc(Config) -> ok. +skipped_tc(_Config) -> ok. -failing_tc_1(Config) -> ?ERROR({error, testcase}). +failing_tc_1(_Config) -> ?ERROR({error, testcase}). -failing_tc_2(Config) -> ok. +failing_tc_2(_Config) -> ok. -failing_tc_3(Config) -> ok. +failing_tc_3(_Config) -> ok. + +eventually_passing_tc(Config) -> + {counter, AtomicRef} = proplists:lookup(counter, Config), + Counter = atomics:add_get(AtomicRef, 1, 1), + if + (Counter rem 3) =:= 0 -> + ok; + true -> + ?ERROR({error, eventually_passing_testcase, Counter}) + end. diff --git a/tools/summarise-ct-results b/tools/summarise-ct-results index 0838cab289..4b0ac2ec33 100755 --- a/tools/summarise-ct-results +++ b/tools/summarise-ct-results @@ -50,10 +50,16 @@ verify_results_with_groups(OkGroups, FailedGroups, EventuallyOkTests, OkGroups =< 0 -> erlang:halt(100405); AutoSkipped > 0 -> + %% this can give false negative result for eventually passing + %% groups with 'sequence' and 'repeat_until_all_ok' flags, but + %% at least it's not false positive result. io:format("Failing the test due to auto skipped cases~n"), erlang:halt(AutoSkipped); - FailedGroups + FailedTests - EventuallyOkTests > 0 -> + FailedGroups > 0 -> io:format("Failing the test due to failed rerun groups~n"), + erlang:halt(FailedGroups); + FailedTests > EventuallyOkTests -> + io:format("Failing the test due to failed test cases~n"), erlang:halt(FailedGroups + FailedTests - EventuallyOkTests); true -> erlang:halt(0) From 6c0f28fb2f903291c51db846ce94735a9c04eda8 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 5 Jul 2023 01:32:22 +0200 Subject: [PATCH 09/18] improving truncated_counter_file reporting at ct_markdown_errors_hook --- big_tests/failure_reporting_testing.spec | 3 +++ big_tests/src/ct_markdown_errors_hook.erl | 21 ++++++------------ big_tests/src/ct_test_hook.erl | 3 +-- big_tests/tests/test_SUITE.erl | 26 +++++++++++------------ 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/big_tests/failure_reporting_testing.spec b/big_tests/failure_reporting_testing.spec index cc56cf1e5e..d626a7e9d8 100644 --- a/big_tests/failure_reporting_testing.spec +++ b/big_tests/failure_reporting_testing.spec @@ -17,6 +17,9 @@ %%% and an overall resulting all_groups.summary file. {ct_hooks, [ct_groups_summary_hook]}. +%%% this hook generates /tmp/ct_summary, /tmp/ct_markdown +%%% and /tmp/ct_markdown_truncated. these files are required +%%% for GH and GA4 CI reporting. {ct_hooks, [ct_markdown_errors_hook]}. %%% this hook prints execution summary in the end of diff --git a/big_tests/src/ct_markdown_errors_hook.erl b/big_tests/src/ct_markdown_errors_hook.erl index 515b229f79..358402383c 100644 --- a/big_tests/src/ct_markdown_errors_hook.erl +++ b/big_tests/src/ct_markdown_errors_hook.erl @@ -76,31 +76,22 @@ handle_return_unsafe(SuiteName, Place, Return, Config, State) -> exec_limited_number_of_times(F, State) end. -exec_limited_number_of_times(_F, State=#state{limit=0, file=_File, - truncated_counter_file = TrFile}) -> +exec_limited_number_of_times(F, State=#state{limit=Limit}) when Limit > 0-> + F(), + State#state{limit=Limit-1}; +exec_limited_number_of_times(_F, State=#state{truncated_counter_file = TrFile, + limit=Limit, file=_File})-> %% Log truncated, increment counter - TrCounter = old_truncated_counter_value(TrFile), + TrCounter = (-1 * Limit), file:write_file(TrFile, integer_to_binary(TrCounter+1)), - State; -exec_limited_number_of_times(F, State=#state{limit=Limit}) -> - F(), State#state{limit=Limit-1}. -old_truncated_counter_value(TrFile) -> - case file:read_file(TrFile) of - {ok, Bin} -> - binary_to_integer(Bin); - _ -> - 0 - end. - log_summary(SuiteName, GroupName, Place, #state{summary_file = SummaryFile}) -> SummaryText = make_summary_text(SuiteName, GroupName, Place), file:write_file(SummaryFile, [SummaryText, $\n], [append]), ok. log_error(SuiteName, GroupName, Place, Error, Config, #state{file = File, summary_file = _SummaryFile}) -> - _MaybeLogLink = make_log_link(Config), LogLink = make_log_link(Config), %% Spoler syntax %% https://github.com/dear-github/dear-github/issues/166 diff --git a/big_tests/src/ct_test_hook.erl b/big_tests/src/ct_test_hook.erl index 44a3c70f38..70a2fcd7fb 100644 --- a/big_tests/src/ct_test_hook.erl +++ b/big_tests/src/ct_test_hook.erl @@ -87,8 +87,7 @@ start_tracing() -> {ok, _Pid} -> dbg:p(all, call), % dbg:tp(?MODULE, []), - % dbg:tpl(ct_markdown_errors_hook, cx), - dbg:tp(ct_groups_summary_hook, cx), + dbg:tp(ct_markdown_errors_hook, cx), ok; {error, already_started} -> ok diff --git a/big_tests/tests/test_SUITE.erl b/big_tests/tests/test_SUITE.erl index 0cc03e145f..9222bb76fc 100644 --- a/big_tests/tests/test_SUITE.erl +++ b/big_tests/tests/test_SUITE.erl @@ -6,19 +6,19 @@ -define(ERROR(E), (error(E))). % -define(ERROR(E), (exit(E))). -all() -> [{group, passing_repeating_group_with_autoskip}]. - % [{group, test_group}, - % {group, skipped_test_group}, - % {group, failing_test_group_1}, - % {group, failing_test_group_2}, - % {group, nested_test_group}, - % {group, deeply_nested_test_group}, - % {group, failing_nested_test_group_1}, - % {group, failing_nested_test_group_2}, - % {group, failing_repeating_group}, - % {group, passing_repeating_group}, - % {group, passing_repeating_group_with_autoskip} - % | test_cases() ]. +all() -> + [{group, test_group}, + {group, skipped_test_group}, + {group, failing_test_group_1}, + {group, failing_test_group_2}, + {group, nested_test_group}, + {group, deeply_nested_test_group}, + {group, failing_nested_test_group_1}, + {group, failing_nested_test_group_2}, + {group, failing_repeating_group}, + {group, passing_repeating_group}, + {group, passing_repeating_group_with_autoskip} + | test_cases() ]. groups() -> [{test_group, [], test_cases()}, From 4b31075fb75536c312c66efefa76109caabad676 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 5 Jul 2023 10:50:14 +0200 Subject: [PATCH 10/18] removing redundant logging from ct_mongoose_hook and ct_tty_hook. ct_mongoose_log_hook already does it, and does it in a better way. --- big_tests/default.spec | 4 ---- big_tests/dynamic_domains.spec | 4 ---- big_tests/mam.spec | 4 ---- big_tests/src/ct_mongoose_hook.erl | 22 +++----------------- big_tests/src/ct_tty_hook.erl | 32 +++++++----------------------- 5 files changed, 10 insertions(+), 56 deletions(-) diff --git a/big_tests/default.spec b/big_tests/default.spec index 561599d3f7..b307b6cb50 100644 --- a/big_tests/default.spec +++ b/big_tests/default.spec @@ -121,7 +121,6 @@ %% ct_tty_hook will log CT failures to TTY verbosely %% ct_mongoose_hook will: -%% * log suite start/end events in the MongooseIM console %% * ensure preset value is passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, @@ -130,6 +129,3 @@ {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} ]}. - -%% To enable printing group and case enters on server side -%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}. diff --git a/big_tests/dynamic_domains.spec b/big_tests/dynamic_domains.spec index 73a309663c..7c73a365d0 100644 --- a/big_tests/dynamic_domains.spec +++ b/big_tests/dynamic_domains.spec @@ -164,7 +164,6 @@ %% ct_tty_hook will log CT failures to TTY verbosely %% ct_mongoose_hook will: -%% * log suite start/end events in the MongooseIM console %% * ensure preset value is passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, @@ -173,6 +172,3 @@ {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} ]}. - -%% To enable printing group and case enters on server side -%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}. diff --git a/big_tests/mam.spec b/big_tests/mam.spec index d9854bff92..73d87e87ec 100644 --- a/big_tests/mam.spec +++ b/big_tests/mam.spec @@ -21,7 +21,6 @@ %% ct_tty_hook will log CT failures to TTY verbosely %% ct_mongoose_hook will: -%% * log suite start/end events in the MongooseIM console %% * ensure preset value is passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, @@ -30,6 +29,3 @@ {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} ]}. - -%% To enable printing group and case enters on server side -%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}. diff --git a/big_tests/src/ct_mongoose_hook.erl b/big_tests/src/ct_mongoose_hook.erl index 7982b8ea7a..fe1c1df98c 100644 --- a/big_tests/src/ct_mongoose_hook.erl +++ b/big_tests/src/ct_mongoose_hook.erl @@ -1,4 +1,4 @@ -%%% @doc Common Test Example Common Test Hook module. +%%% @doc this hook performs server purity check after every suite -module(ct_mongoose_hook). %% @doc Add the following line in your *.spec file @@ -30,20 +30,15 @@ -import(distributed_helper, [mim/0, rpc/4]). --record(state, {print_group, print_case}). - %% @doc Return a unique id for this CTH. id(_Opts) -> "ct_mongoose_hook_001". %% @doc Always called before any other callback function. Use this to initiate %% any common state. -init(_Id, Opts) -> - Unfolded = proplists:unfold(Opts), - PrintGroup = proplists:get_value(print_group, Unfolded, false), - PrintCase = proplists:get_value(print_case, Unfolded, false), +init(_Id, _Opts) -> domain_helper:insert_configured_domains(), - {ok, #state{print_group = PrintGroup, print_case = PrintCase }}. + {ok, #{}}. %% @doc Called before init_per_suite is called. pre_init_per_suite(Suite,Config,State) -> @@ -52,7 +47,6 @@ pre_init_per_suite(Suite,Config,State) -> _ -> undefined end, NewConfig = [{preset, Preset} | Config], - maybe_print_on_server(true, "SUITE", Suite, "starting"), {NewConfig, State}. %% @doc Called after init_per_suite. @@ -65,13 +59,11 @@ pre_end_per_suite(_Suite,Config,State) -> %% @doc Called after end_per_suite. post_end_per_suite(Suite, Config, Return, State) -> - maybe_print_on_server(true, "SUITE", Suite, "finished"), check_server_purity(Suite, Config), {Return, State}. %% @doc Called before each init_per_group. pre_init_per_group(Group,Config,State) -> - maybe_print_on_server(State#state.print_group, "GROUP", Group, "starting"), {Config, State}. %% @doc Called after each init_per_group. @@ -84,17 +76,14 @@ pre_end_per_group(_Group,Config,State) -> %% @doc Called after each end_per_group. post_end_per_group(Group,_Config,Return,State) -> - maybe_print_on_server(State#state.print_group, "GROUP", Group, "finished"), {Return, State}. %% @doc Called before each test case. pre_init_per_testcase(TC,Config,State) -> - maybe_print_on_server(State#state.print_case, "TEST CASE", TC, "starting"), {Config, State}. %% @doc Called after each test case. post_end_per_testcase(TC,_Config,Return,State) -> - maybe_print_on_server(State#state.print_case, "TEST CASE", TC, "finished"), {Return, State}. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, @@ -112,11 +101,6 @@ terminate(_State) -> domain_helper:delete_configured_domains(), ok. -maybe_print_on_server(false, _, _, _) -> - ok; -maybe_print_on_server(true, Event, EventName, EvenType) -> - rpc(mim(), logger, warning, ["====== ~s ~p ~s", [Event, EventName, EvenType]]). - check_server_purity(Suite, Config) -> case escalus_server:name(Config) of mongooseim -> diff --git a/big_tests/src/ct_tty_hook.erl b/big_tests/src/ct_tty_hook.erl index 357ba83c8a..da148dceb8 100644 --- a/big_tests/src/ct_tty_hook.erl +++ b/big_tests/src/ct_tty_hook.erl @@ -26,7 +26,7 @@ -export([on_tc_skip/3]). -export([terminate/1]). --record(state, { total, suite_total, ts, tcs, data, print_group, print_case }). +-record(state, { total, suite_total, ts, tcs, data }). -import(distributed_helper, [mim/0, rpc/4]). @@ -37,12 +37,8 @@ id(_Opts) -> %% @doc Always called before any other callback function. Use this to initiate %% any common state. -init(_Id, Opts) -> - Unfolded = proplists:unfold(Opts), - PrintGroup = proplists:get_value(print_group, Unfolded, false), - PrintCase = proplists:get_value(print_case, Unfolded, false), - {ok, #state{ total = 0, data = [], - print_group = PrintGroup, print_case = PrintCase }}. +init(_Id, _Opts) -> + {ok, #state{ total = 0, data = []}}. %% @doc Called before init_per_suite is called. pre_init_per_suite(_Suite, Config, State) -> @@ -64,7 +60,6 @@ post_end_per_suite(Suite,_Config,Return,State) -> %% @doc Called before each init_per_group. pre_init_per_group(Group,Config,State) -> - print_group_enter(Group, State, "Starting"), {Config, State}. %% @doc Called after each init_per_group. @@ -77,13 +72,11 @@ pre_end_per_group(_Group,Config,State) -> %% @doc Called after each end_per_group. post_end_per_group(Group,_Config,Return,State) -> - print_group_enter(Group, State, "Finished"), {Return, State}. %% @doc Called before each test case. pre_init_per_testcase(TC,Config,State = #state{suite_total = Total}) when is_integer(Total) -> - print_case_enter(TC, State, "Starting"), {Config, State#state{ ts = os:timestamp(), total = Total + 1 } }; pre_init_per_testcase(_TC, Config, State) -> {Config, State}. @@ -94,7 +87,6 @@ post_end_per_testcase(TC, _Config, Return, State) -> %%% this fails when running in parallel: %%% timer:now_diff(os:timestamp(), State#state.ts), TCInfo = {testcase, TC, Return, ParallelTestDiffOverride}, - print_case_enter(TC, State, "Finished"), {Return, State#state{ts = undefined, tcs = [TCInfo | State#state.tcs]}}. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, @@ -136,21 +128,11 @@ print_suite_ok(Name, OkCount) -> io:format("~n====== Suite OK: ~p (All ~p tests passed)~n", [Name, OkCount]). print_suite_failed(Name, OkCount, FailCount, Cases) -> - ct:pal("~n====== Suite FAILED: ~p (~p of ~p tests failed)~n", - [Name, FailCount, OkCount+FailCount]), + ct:print("~n====== Suite FAILED: ~p (~p of ~p tests failed)~n", + [Name, FailCount, OkCount+FailCount]), lists:foreach(fun maybe_print_test_case/1, Cases). maybe_print_test_case({testcase, _Name,ok,_}) -> ok; maybe_print_test_case({testcase, Name,{error, Content},_}) -> - io:format("~n====== Test name: ~p", [Name]), - io:format("~n====== Reason: ~p~n", [Content]). - -print_group_enter(Group, #state{print_group = true}, Msg) -> - rpc(mim(), logger, warning, ["====== ~s GROUP ~p", [Msg, Group]]); -print_group_enter(_Group, _State, _Msg) -> - ok. - -print_case_enter(Group, #state{print_case = true}, Msg) -> - rpc(mim(), logger, warning, ["====== ~s CASE ~p", [Msg, Group]]); -print_case_enter(_Group, _State, _Msg) -> - ok. + ct:print("~n====== Test name: ~p", [Name]), + ct:print("~n====== Reason: ~p~n", [Content]). From 61140cdf4604e0da3f21be0c919bd06f36b36e50 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Thu, 6 Jul 2023 15:25:35 +0200 Subject: [PATCH 11/18] removing ct_mim_config_hook it's quite an overhead to have an extra hook just to set the mim_data_dir parameter, so the hook is merged with ct_mongoose_hook --- big_tests/default.spec | 3 +-- big_tests/dynamic_domains.spec | 3 +-- big_tests/mam.spec | 3 +-- big_tests/src/ct_mim_config_hook.erl | 34 ---------------------------- big_tests/src/ct_mongoose_hook.erl | 3 ++- 5 files changed, 5 insertions(+), 41 deletions(-) delete mode 100644 big_tests/src/ct_mim_config_hook.erl diff --git a/big_tests/default.spec b/big_tests/default.spec index b307b6cb50..a3e52572b6 100644 --- a/big_tests/default.spec +++ b/big_tests/default.spec @@ -121,10 +121,9 @@ %% ct_tty_hook will log CT failures to TTY verbosely %% ct_mongoose_hook will: -%% * ensure preset value is passed to ct Config +%% * ensure preset & mim_data_dir values are passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, - ct_mim_config_hook, ct_markdown_errors_hook, {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} diff --git a/big_tests/dynamic_domains.spec b/big_tests/dynamic_domains.spec index 7c73a365d0..0a4867274a 100644 --- a/big_tests/dynamic_domains.spec +++ b/big_tests/dynamic_domains.spec @@ -164,10 +164,9 @@ %% ct_tty_hook will log CT failures to TTY verbosely %% ct_mongoose_hook will: -%% * ensure preset value is passed to ct Config +%% * ensure preset & mim_data_dir values are passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, - ct_mim_config_hook, ct_markdown_errors_hook, {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} diff --git a/big_tests/mam.spec b/big_tests/mam.spec index 73d87e87ec..43c5659bd1 100644 --- a/big_tests/mam.spec +++ b/big_tests/mam.spec @@ -21,11 +21,10 @@ %% ct_tty_hook will log CT failures to TTY verbosely %% ct_mongoose_hook will: -%% * ensure preset value is passed to ct Config +%% * ensure preset & mim_data_dir values are passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, ct_markdown_errors_hook, - ct_mim_config_hook, {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} ]}. diff --git a/big_tests/src/ct_mim_config_hook.erl b/big_tests/src/ct_mim_config_hook.erl deleted file mode 100644 index 9362db43a7..0000000000 --- a/big_tests/src/ct_mim_config_hook.erl +++ /dev/null @@ -1,34 +0,0 @@ --module(ct_mim_config_hook). - -%% @doc Add the following line in your *.spec file to enable -%% fixes for Common Tests for your common tests: -%% {ct_hooks, [ct_mim_config_hook]}. - -%% Callbacks --export([id/1]). --export([init/2]). --export([pre_init_per_suite/3]). --record(state, { }). - -%% @doc Return a unique id for this CTH. -id(_Opts) -> - ?MODULE_STRING ++ "_001". - -%% @doc Always called before any other callback function. Use this to initiate -%% any common state. -init(_Id, _Opts) -> - {ok, #state{ }}. - -%% Fix data_dir for init_per_suite function -pre_init_per_suite(SuiteName, Config, State) -> - {fix_data_dir(SuiteName, Config), State}. - -%% Common tests set data dir based on beam source file attribute. -%% Which makes us care, where the file has been compiled and makes -%% us less portable. -%% Lets set out own data_dir. -%% Be aware, CT would try to set data_dir over and over again, -%% so to be safe we use new mim_data_dir option instead. -fix_data_dir(SuiteName, Config) -> - DataDir = path_helper:data_dir(SuiteName, Config), - [{mim_data_dir, DataDir}|Config]. diff --git a/big_tests/src/ct_mongoose_hook.erl b/big_tests/src/ct_mongoose_hook.erl index fe1c1df98c..921adfe111 100644 --- a/big_tests/src/ct_mongoose_hook.erl +++ b/big_tests/src/ct_mongoose_hook.erl @@ -46,7 +46,8 @@ pre_init_per_suite(Suite,Config,State) -> {ok, Value} -> Value; _ -> undefined end, - NewConfig = [{preset, Preset} | Config], + DataDir = path_helper:data_dir(Suite, Config), + NewConfig = [{preset, Preset}, {mim_data_dir, DataDir} | Config], {NewConfig, State}. %% @doc Called after init_per_suite. From 30b5798dc8b2e4e1c8160695bb69511962534e03 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Thu, 6 Jul 2023 17:27:50 +0200 Subject: [PATCH 12/18] removing dummy optional CT hook callbacks from big_tests hooks --- big_tests/src/ct_mongoose_log_hook.erl | 39 -------------------------- big_tests/src/ct_tty_hook.erl | 34 ++-------------------- 2 files changed, 2 insertions(+), 71 deletions(-) diff --git a/big_tests/src/ct_mongoose_log_hook.erl b/big_tests/src/ct_mongoose_log_hook.erl index 3776450abd..b4de82b821 100644 --- a/big_tests/src/ct_mongoose_log_hook.erl +++ b/big_tests/src/ct_mongoose_log_hook.erl @@ -13,22 +13,14 @@ -export([init/2]). -export([pre_init_per_suite/3]). --export([post_init_per_suite/4]). -export([pre_end_per_suite/3]). --export([post_end_per_suite/4]). -export([pre_init_per_group/3]). --export([post_init_per_group/4]). -export([pre_end_per_group/3]). --export([post_end_per_group/4]). -export([pre_init_per_testcase/3]). -export([post_end_per_testcase/4]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). - --export([terminate/1]). -record(state, { node, cookie, reader, writer, current_line_num, out_file, url_file, group, suite, priv_dir }). @@ -49,34 +41,18 @@ init(_Id, [Node0, Cookie0]) -> pre_init_per_suite(Suite,Config,State) -> {Config, State#state{group=no_group, suite=Suite}}. -%% @doc Called after init_per_suite. -post_init_per_suite(_Suite,_Config,Return,State) -> - {Return, State}. - %% @doc Called before end_per_suite. pre_end_per_suite(_Suite,Config,State) -> {Config, State#state{suite=no_suite}}. -%% @doc Called after end_per_suite. -post_end_per_suite(_Suite,_Config,Return,State) -> - {Return, State}. - %% @doc Called before each init_per_group. pre_init_per_group(Group,Config,State) -> {Config, State#state{group=Group}}. -%% @doc Called after each init_per_group. -post_init_per_group(_Group,_Config,Return,State) -> - {Return, State}. - %% @doc Called after each end_per_group. pre_end_per_group(_Group,Config,State) -> {Config, State#state{group=no_group}}. -%% @doc Called after each end_per_group. -post_end_per_group(_Group,_Config,Return,State) -> - {Return, State}. - %% @doc Called before each test case. pre_init_per_testcase(TC,Config,State=#state{}) -> Dog = test_server:timetrap(test_server:seconds(10)), @@ -93,21 +69,6 @@ post_end_per_testcase(TC,_Config,Return,State) -> test_server:timetrap_cancel(Dog), {Return, State2 }. -%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, -%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. -on_tc_fail(_TC, _Reason, State) -> - State. - -%% @doc Called when a test case is skipped by either user action -%% or due to an init function failing. -on_tc_skip(_TC, _Reason, State) -> - State. - -%% @doc Called when the scope of the CTH is done -terminate(_State) -> - ok. - - % -------------------------------------------- spawn_log_reader(Node, Cookie) -> diff --git a/big_tests/src/ct_tty_hook.erl b/big_tests/src/ct_tty_hook.erl index da148dceb8..960a99c6ee 100644 --- a/big_tests/src/ct_tty_hook.erl +++ b/big_tests/src/ct_tty_hook.erl @@ -10,15 +10,8 @@ -export([init/2]). -export([pre_init_per_suite/3]). --export([post_init_per_suite/4]). --export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). - -export([pre_init_per_testcase/3]). -export([post_end_per_testcase/4]). @@ -26,6 +19,7 @@ -export([on_tc_skip/3]). -export([terminate/1]). + -record(state, { total, suite_total, ts, tcs, data }). -import(distributed_helper, [mim/0, @@ -44,36 +38,12 @@ init(_Id, _Opts) -> pre_init_per_suite(_Suite, Config, State) -> {Config, State#state{ suite_total = 0, tcs = [] }}. -%% @doc Called after init_per_suite. -post_init_per_suite(_Suite,_Config,Return,State) -> - {Return, State}. - -%% @doc Called before end_per_suite. -pre_end_per_suite(_Suite,Config,State) -> - {Config, State}. - %% @doc Called after end_per_suite. post_end_per_suite(Suite,_Config,Return,State) -> Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)}, {Return, State#state{ data = [Data | State#state.data] , total = State#state.total + State#state.suite_total } }. -%% @doc Called before each init_per_group. -pre_init_per_group(Group,Config,State) -> - {Config, State}. - -%% @doc Called after each init_per_group. -post_init_per_group(_Group,_Config,Return,State) -> - {Return, State}. - -%% @doc Called after each end_per_group. -pre_end_per_group(_Group,Config,State) -> - {Config, State}. - -%% @doc Called after each end_per_group. -post_end_per_group(Group,_Config,Return,State) -> - {Return, State}. - %% @doc Called before each test case. pre_init_per_testcase(TC,Config,State = #state{suite_total = Total}) when is_integer(Total) -> @@ -118,7 +88,7 @@ aggregate_results(Cases) -> Fails = [ C || {testcase, _, {error, _}, _} = C <- Cases ], {length(Oks), length(Fails)}. -print_suite({suites, Name,_, TestCases}) -> +print_suite({suites, Name, _, TestCases}) -> case aggregate_results(TestCases) of {Oks, 0} -> print_suite_ok(Name, Oks); {Oks, Fails} -> print_suite_failed(Name,Oks,Fails,TestCases) From d6b34200578e93bb52c224501af8571eb0452323 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Thu, 6 Jul 2023 17:35:20 +0200 Subject: [PATCH 13/18] minor improvements to ct_tty_hook --- big_tests/src/ct_progress_hook.erl | 3 +-- big_tests/src/ct_tty_hook.erl | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/big_tests/src/ct_progress_hook.erl b/big_tests/src/ct_progress_hook.erl index dc3f2bab6b..b54d957f14 100644 --- a/big_tests/src/ct_progress_hook.erl +++ b/big_tests/src/ct_progress_hook.erl @@ -14,7 +14,6 @@ -export([post_end_per_suite/4, post_end_per_group/4, post_end_per_testcase/4]). --record(state, { }). %% @doc Return a unique id for this CTH. id(_Opts) -> @@ -23,7 +22,7 @@ id(_Opts) -> %% @doc Always called before any other callback function. Use this to initiate %% any common state. init(_Id, _Opts) -> - {ok, #state{ }}. + {ok, #{ }}. post_init_per_suite(_SuiteName, _Config, Return, State) -> handle_return(Return, false), diff --git a/big_tests/src/ct_tty_hook.erl b/big_tests/src/ct_tty_hook.erl index 960a99c6ee..e0e0e8153a 100644 --- a/big_tests/src/ct_tty_hook.erl +++ b/big_tests/src/ct_tty_hook.erl @@ -21,6 +21,7 @@ -export([terminate/1]). -record(state, { total, suite_total, ts, tcs, data }). +-record(suite, { name, total, tcs }). -import(distributed_helper, [mim/0, rpc/4]). @@ -40,7 +41,8 @@ pre_init_per_suite(_Suite, Config, State) -> %% @doc Called after end_per_suite. post_end_per_suite(Suite,_Config,Return,State) -> - Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)}, + Data = #suite{ name = Suite, total = State#state.suite_total, + tcs = lists:reverse(State#state.tcs) }, {Return, State#state{ data = [Data | State#state.data] , total = State#state.total + State#state.suite_total } }. @@ -88,7 +90,7 @@ aggregate_results(Cases) -> Fails = [ C || {testcase, _, {error, _}, _} = C <- Cases ], {length(Oks), length(Fails)}. -print_suite({suites, Name, _, TestCases}) -> +print_suite(#suite{name=Name, tcs = TestCases}) -> case aggregate_results(TestCases) of {Oks, 0} -> print_suite_ok(Name, Oks); {Oks, Fails} -> print_suite_failed(Name,Oks,Fails,TestCases) From adec35ae9d0465bcf02108c6efb5bf61336e25e4 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Thu, 6 Jul 2023 17:44:20 +0200 Subject: [PATCH 14/18] moving halting logic from ct_tty_hook:on_tc_skip/3 to ct_groups_summary_hook:on_tc_skip/3 --- big_tests/src/ct_groups_summary_hook.erl | 11 +++++++++++ big_tests/src/ct_tty_hook.erl | 10 ---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/big_tests/src/ct_groups_summary_hook.erl b/big_tests/src/ct_groups_summary_hook.erl index 76655b7f88..5c34711c55 100644 --- a/big_tests/src/ct_groups_summary_hook.erl +++ b/big_tests/src/ct_groups_summary_hook.erl @@ -10,8 +10,10 @@ pre_init_per_suite/3, post_end_per_suite/4, post_end_per_group/4, + on_tc_skip/3, terminate/1]). + -include_lib("common_test/include/ct.hrl"). -record(group_status, {status = ok :: ok | failed, @@ -51,6 +53,15 @@ post_end_per_group(GroupName, Config, Return, State) -> ct:log("NewState: ~p", [State1]), {Return, State1}. +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. +on_tc_skip(_TC, Reason, State) -> + case Reason of + %% Just quit if files were not build correctly + {tc_user_skip, "Make failed"} -> erlang:halt(1); + _ -> State + end. + terminate(#{total_ok := OK, total_eventually_ok_tests := TotalEventuallyOK, total_failed := TotalFailed} = State) -> Content = io_lib:format("~p.~n~p.~n~p~n.", diff --git a/big_tests/src/ct_tty_hook.erl b/big_tests/src/ct_tty_hook.erl index e0e0e8153a..9d78ba3305 100644 --- a/big_tests/src/ct_tty_hook.erl +++ b/big_tests/src/ct_tty_hook.erl @@ -16,7 +16,6 @@ -export([post_end_per_testcase/4]). -export([on_tc_fail/3]). --export([on_tc_skip/3]). -export([terminate/1]). @@ -67,15 +66,6 @@ on_tc_fail(TC, Reason, State) -> ct:print("~p~n~p", [TC, Reason]), State. -%% @doc Called when a test case is skipped by either user action -%% or due to an init function failing. -on_tc_skip(_TC, Reason, State) -> - case Reason of - %% Just quit if files were not build correctly - {tc_user_skip, "Make failed"} -> erlang:halt(1); - _ -> State - end. - %% @doc Called when the scope of the CTH is done terminate(State) -> lists:foreach(fun (SuiteData) -> From 1547306f0ac3ab9af49b614c098a77a68e58c051 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Fri, 7 Jul 2023 13:48:58 +0200 Subject: [PATCH 15/18] adding optional CT suite/group/testcase logging at MIM node --- big_tests/default.spec | 9 ++- big_tests/dynamic_domains.spec | 9 ++- big_tests/mam.spec | 9 ++- big_tests/src/ct_mongoose_log_hook.erl | 104 +++++++++++++++++++------ 4 files changed, 94 insertions(+), 37 deletions(-) diff --git a/big_tests/default.spec b/big_tests/default.spec index a3e52572b6..0aea713175 100644 --- a/big_tests/default.spec +++ b/big_tests/default.spec @@ -124,7 +124,8 @@ %% * ensure preset & mim_data_dir values are passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, - ct_markdown_errors_hook, - {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, - {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} - ]}. + ct_markdown_errors_hook, ct_mongoose_log_hook]}. + +%% since test-runner.sh can be executed with the --one-node option, +%% log collection is enabled by default for host mim1 only. +% {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}, {log, []}]}]}. diff --git a/big_tests/dynamic_domains.spec b/big_tests/dynamic_domains.spec index 0a4867274a..aed34e1a66 100644 --- a/big_tests/dynamic_domains.spec +++ b/big_tests/dynamic_domains.spec @@ -167,7 +167,8 @@ %% * ensure preset & mim_data_dir values are passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, - ct_markdown_errors_hook, - {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, - {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} - ]}. + ct_markdown_errors_hook, ct_mongoose_log_hook]}. + +%% since test-runner.sh can be executed with the --one-node option, +%% log collection is enabled by default for host mim1 only. +% {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}, {log, []}]}]}. diff --git a/big_tests/mam.spec b/big_tests/mam.spec index 43c5659bd1..0183cb7862 100644 --- a/big_tests/mam.spec +++ b/big_tests/mam.spec @@ -24,7 +24,8 @@ %% * ensure preset & mim_data_dir values are passed to ct Config %% * check server's purity after SUITE {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook, - ct_markdown_errors_hook, - {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]}, - {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]} - ]}. + ct_markdown_errors_hook, ct_mongoose_log_hook]}. + +%% since test-runner.sh can be executed with the --one-node option, +%% log collection is enabled by default for host mim1 only. +% {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}, {log, []}]}]}. diff --git a/big_tests/src/ct_mongoose_log_hook.erl b/big_tests/src/ct_mongoose_log_hook.erl index b4de82b821..719276ac57 100644 --- a/big_tests/src/ct_mongoose_log_hook.erl +++ b/big_tests/src/ct_mongoose_log_hook.erl @@ -1,60 +1,78 @@ %%% @doc This hook copies mongooseim.log into an html file (with labels), -%%% and inserts links into test case specific html report files. note that +%%% and inserts links into test case specific html report files. Note that %%% a temporary html file is created for each link in log_private directory, %%% for more information see comments for add_log_link_to_line/5 function. +%%% +%%% Hook options must be in proplist format: +%%% * host - atom, one of the hosts provided in the common test config file. +%%% Optional parameter, default value: 'mim' +%%% * log - list of atoms (suite, group or testcase). +%%% Optional parameter, default value: [suite] +%%% +%%% examples of the *.spec file configuration: +%%% * {ct_hooks, [ct_mongoose_log_hook]}. +%%% * {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}]}]}. +%%% * {ct_hooks, [{ct_mongoose_log_hook,[{host, mim3}, {log, [suite, group]}]}]}. +%%% * {ct_hooks, [{ct_mongoose_log_hook,[{host, fed}, {log, [testcase]}]}]}. +%%% * {ct_hooks, [{ct_mongoose_log_hook,[{host, reg}, {log, []}]}]}. +%%% -module(ct_mongoose_log_hook). -%% @doc Add the following line in your *.spec file to -%% copy mongooseim.log into CT reports: -%% {ct_hooks, [ct_mongoose_log_hook]}. %% Callbacks -export([id/1]). -export([init/2]). +-export([terminate/1]). -export([pre_init_per_suite/3]). --export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). -export([pre_init_per_group/3]). --export([pre_end_per_group/3]). +-export([post_end_per_group/4]). -export([pre_init_per_testcase/3]). -export([post_end_per_testcase/4]). --record(state, { node, cookie, reader, writer, +-record(state, { node_name, reader, writer, current_line_num, out_file, url_file, group, suite, - priv_dir }). + priv_dir, log_flags = [] }). -include_lib("exml/include/exml.hrl"). %% @doc Return a unique id for this CTH. -id([Node, _Cookie]) -> - "ct_mongoose_log_hook_" ++ atom_to_list(Node). +id(Opts) -> + Host = proplists:get_value(host, Opts, mim), + "ct_mongoose_log_hook_" ++ atom_to_list(Host). %% @doc Always called before any other callback function. Use this to initiate %% any common state. -init(_Id, [Node0, Cookie0]) -> - Node = ct:get_config(Node0), - Cookie = ct:get_config(Cookie0), - {ok, #state{ node=Node, cookie=Cookie }}. +init(_Id, Opts) -> + Node = connect_mim_node(Opts), + LogFlags = proplists:get_value(log, Opts, [suite]), + {ok, #state{ node_name=Node, log_flags=LogFlags }}. %% @doc Called before init_per_suite is called. pre_init_per_suite(Suite,Config,State) -> + maybe_print_log_on_mim_node(suite, starting, Suite, State), {Config, State#state{group=no_group, suite=Suite}}. %% @doc Called before end_per_suite. -pre_end_per_suite(_Suite,Config,State) -> - {Config, State#state{suite=no_suite}}. +post_end_per_suite(Suite,_Config,Return,State) -> + maybe_print_log_on_mim_node(suite, finishing, Suite, State), + {Return, State#state{suite=no_suite}}. %% @doc Called before each init_per_group. pre_init_per_group(Group,Config,State) -> + maybe_print_log_on_mim_node(group, starting, Group, State), {Config, State#state{group=Group}}. %% @doc Called after each end_per_group. -pre_end_per_group(_Group,Config,State) -> - {Config, State#state{group=no_group}}. +post_end_per_group(Group,_Config,Return,State) -> + maybe_print_log_on_mim_node(group, finishing, Group, State), + {Return, State#state{group=no_group}}. %% @doc Called before each test case. pre_init_per_testcase(TC,Config,State=#state{}) -> + maybe_print_log_on_mim_node(testcase, starting, TC, State), Dog = test_server:timetrap(test_server:seconds(10)), State2 = keep_priv_dir(Config, State), State3 = ensure_initialized(Config, State2), @@ -67,13 +85,16 @@ post_end_per_testcase(TC,_Config,Return,State) -> Dog = test_server:timetrap(test_server:seconds(10)), State2 = post_insert_line_numbers_into_report(State, TC), test_server:timetrap_cancel(Dog), + maybe_print_log_on_mim_node(testcase, finishing, TC, State), {Return, State2 }. +%% @doc Called when the scope of the CTH is done +terminate(State) -> + insert_line_numbers_into_report(State). + % -------------------------------------------- -spawn_log_reader(Node, Cookie) -> - %% Set cookie permanently - erlang:set_cookie(Node, Cookie), +spawn_log_reader(Node) -> AbsName = rpc:call(Node, filename, absname, ["log/mongooseim.log.1"], 5000), case is_list(AbsName) of true -> @@ -126,7 +147,7 @@ make_link_name(Line) when is_integer(Line) -> make_content(CurrentLineNum, Line) -> << (list_to_binary(integer_to_list(CurrentLineNum)))/binary, " ", Line/binary>>. -ensure_initialized(Config, State=#state{node=Node, cookie=Cookie, out_file=undefined}) -> +ensure_initialized(Config, State=#state{node_name=Node, out_file=undefined}) -> RunDir = path_helper:ct_run_dir(Config), File = atom_to_list(Node) ++ ".log.html", %% On disk @@ -134,7 +155,7 @@ ensure_initialized(Config, State=#state{node=Node, cookie=Cookie, out_file=undef %% In browser UrlFile = ct_logs:uri(filename:join(path_helper:ct_run_dir_in_browser(Config), File)), try - {ok, Reader} = spawn_log_reader(Node, Cookie), + {ok, Reader} = spawn_log_reader(Node), %% self() process is temporary {ok, Writer} = open_out_file(OutFile), file:write(Writer, "
"),
@@ -159,7 +180,7 @@ keep_priv_dir(Config, State) ->
 
 pre_insert_line_numbers_into_report(State=#state{writer=undefined}, _TC) ->
     State; % Invalid state
-pre_insert_line_numbers_into_report(State=#state{node=Node, reader=Reader, writer=Writer,
+pre_insert_line_numbers_into_report(State=#state{node_name=Node, reader=Reader, writer=Writer,
                                              current_line_num=CurrentLineNum, url_file=UrlFile,
                                              priv_dir=PrivDir, group=Group, suite=Suite}, TC) ->
     CurrentLineNum2 = read_and_write_lines(Node, Reader, Writer, CurrentLineNum),
@@ -172,7 +193,7 @@ pre_insert_line_numbers_into_report(State=#state{node=Node, reader=Reader, write
 
 post_insert_line_numbers_into_report(State=#state{writer=undefined}, _TC) ->
     State; % Invalid state
-post_insert_line_numbers_into_report(State=#state{node=Node, reader=Reader, writer=Writer,
+post_insert_line_numbers_into_report(State=#state{node_name=Node, reader=Reader, writer=Writer,
                                              current_line_num=CurrentLineNum, url_file=UrlFile,
                                              group=Group, suite=Suite, priv_dir=PrivDir}, TC) ->
     CurrentLineNum2 = read_and_write_lines(Node, Reader, Writer, CurrentLineNum),
@@ -184,6 +205,11 @@ post_insert_line_numbers_into_report(State=#state{node=Node, reader=Reader, writ
     file:write(Writer, Message),
     State#state{current_line_num=CurrentLineNum2}.
 
+insert_line_numbers_into_report(State=#state{node_name=Node, reader=Reader, writer=Writer,
+                                             current_line_num=CurrentLineNum}) ->
+    CurrentLineNum2 = read_and_write_lines(Node, Reader, Writer, CurrentLineNum),
+    State#state{current_line_num=CurrentLineNum2}.
+
 %% Function `escalus_ct:add_log_link(Heading, URL, Type).'
 %% allows to add simple links.
 %%
@@ -261,3 +287,31 @@ compare_host_names(Node1, Node2) ->
 node_to_host(Node) when is_atom(Node) ->
     [_Name, Host] = string:tokens(atom_to_list(Node), "@"),
     list_to_atom(Host).
+
+connect_mim_node(HookOpts) ->
+    Host = proplists:get_value(host, HookOpts, mim),
+    Node = ct:get_config({hosts, Host, node}),
+    Cookie = ct:get_config(ejabberd_cookie),
+    %% Set cookie permanently
+    erlang:set_cookie(Node, Cookie),
+    %% this log message lands at misc_io.log.html file
+    ct:pal("connecting to the '~p' node (cookie: '~p')", [Node, Cookie]),
+    %% crash if cannot connect to the node
+    true = net_kernel:connect_node(Node),
+    Node.
+
+maybe_print_log_on_mim_node(Type, Event, Name, #state{log_flags = LogFlags, node_name = Node}) ->
+    ValidEvents = [starting, finishing],
+    ValidTypes = [suite, group, testcase],
+    case {lists:member(Type, LogFlags),
+          lists:member(Event, ValidEvents),
+          lists:member(Type, ValidTypes)} of
+        {_, _, false} ->
+            ct:pal("Invalid logging type: ~p", [Type]);
+        {_, false, _} ->
+            ct:pal("Invalid logging event: ~p", [Event]);
+        {true, _, _} ->
+            rpc:call(Node, logger, warning, ["====== ~p ~p ~p", [Event, Name, Type]]);
+        _ ->
+            ok
+    end.

From 5bbe8ddf90f6822fe86c698ff298a64e01b57405 Mon Sep 17 00:00:00 2001
From: Denys Gonchar 
Date: Mon, 10 Jul 2023 17:30:09 +0200
Subject: [PATCH 16/18] updating pkg GH Actions job

testing package building for ubuntu_xenial as it's not tested by CircleCI
---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6e2f063665..384a9fe422 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -172,10 +172,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pkg: [centos_7, debian_stretch]
+        pkg: [centos_7, ubuntu_xenial]
     runs-on: ubuntu-22.04
     env:
-      ESL_ERLANG_PKG_VER: "25.0.1-1"
+      ESL_ERLANG_PKG_VER: "25.0.3-1"
       pkg_PLATFORM: ${{matrix.pkg}}
     steps:
       - uses: actions/checkout@v3

From ea1c44d3a09efbeb07311b5ce06a2ac5bcfc0f1e Mon Sep 17 00:00:00 2001
From: Denys Gonchar 
Date: Tue, 11 Jul 2023 12:31:51 +0200
Subject: [PATCH 17/18] report failed test cases to GA4 only for master branch

---
 .circleci/template.yml   | 14 +++++++++-----
 .github/workflows/ci.yml |  4 ++--
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.circleci/template.yml b/.circleci/template.yml
index 9bc226581a..7c5729cfb6 100644
--- a/.circleci/template.yml
+++ b/.circleci/template.yml
@@ -304,11 +304,15 @@ commands:
               if [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then tools/circleci-upload-to-s3.sh; fi
   report_failed_test_cases_to_ga4:
     steps:
-      - run:
-          name: Report failed test cases
-          when: always
-          command: |
-              tools/gh-report-failing-testcases-to-ga4.sh
+      - when:
+          condition:
+            equal: [ master, << pipeline.git.branch >> ]
+          steps:
+            - run:
+                name: Report failed test cases to GA4
+                when: always
+                command: |
+                  tools/gh-report-failing-testcases-to-ga4.sh
   publish_github_comment:
     steps:
       - run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 384a9fe422..ae6d7d8436 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -84,7 +84,7 @@ jobs:
         if: ${{ failure() }}
         run: tools/gh-upload-to-s3.sh big_tests/ct_report
       - name: upload big_tests results to GA4
-        if: ${{ always() }}
+        if: github.ref_name == 'master'
         run: tools/gh-report-failing-testcases-to-ga4.sh
 
   dynamic_domains_big_tests:
@@ -112,7 +112,7 @@ jobs:
         if: ${{ failure() }}
         run: tools/gh-upload-to-s3.sh big_tests/ct_report
       - name: upload big_tests results to GA4
-        if: ${{ always() }}
+        if: github.ref_name == 'master'
         run: tools/gh-report-failing-testcases-to-ga4.sh
 
   coveralls_webhook:

From 15d30b5199f4fa15e10348d690d20c206c2051f7 Mon Sep 17 00:00:00 2001
From: Denys Gonchar 
Date: Wed, 12 Jul 2023 11:12:47 +0200
Subject: [PATCH 18/18] fixing failures in
 metrics_roster_SUITE:end_per_testcase/2

---
 big_tests/tests/metrics_roster_SUITE.erl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/big_tests/tests/metrics_roster_SUITE.erl b/big_tests/tests/metrics_roster_SUITE.erl
index febd955d7a..c2502a0950 100644
--- a/big_tests/tests/metrics_roster_SUITE.erl
+++ b/big_tests/tests/metrics_roster_SUITE.erl
@@ -295,7 +295,9 @@ add_sample_contact(Alice, Bob, Groups, Name) ->
 remove_roster(Config, UserSpec) ->
     [Username, Server, _Pass] = escalus_users:get_usp(Config, UserSpec),
     Acc = mongoose_helper:new_mongoose_acc(Server),
-    rpc(mim(), mod_roster, remove_user, [Acc, Username, Server]).
+    Extra = #{host_type => domain_helper:host_type()},
+    Params = #{jid => jid:make_bare(Username, Server)},
+    rpc(mim(), mod_roster, remove_user, [Acc, Params, Extra]).
 
 mongoose_metrics(ConfigIn, Metrics) ->
     Predefined = proplists:get_value(mongoose_metrics, ConfigIn, []),