Reporting to ga4

.circleci/template.yml

@@ -302,6 +302,17 @@ commands:
           command: |
               tools/circleci-prepare-log-dir.sh
               if [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then tools/circleci-upload-to-s3.sh; fi
+  report_failed_test_cases_to_ga4:
+    steps:
+      - when:
+          condition:
+            equal: [ master, << pipeline.git.branch >> ]
+          steps:
+            - run:
+                name: Report failed test cases to GA4
+                when: always
+                command: |
+                  tools/gh-report-failing-testcases-to-ga4.sh
   publish_github_comment:
     steps:
       - run:
@@ -630,6 +641,7 @@ jobs:
             tail -100 _build/mim1/rel/mongooseim/log/mongooseim.log.1
       - upload_results_to_aws
       - publish_github_comment
+      - report_failed_test_cases_to_ga4
 
   dialyzer:
     executor: << parameters.executor >>
@@ -943,6 +955,7 @@ workflows:
           filters: *all_tags
       - big_tests_in_docker:
           name: elasticsearch_and_cassandra_25
+          spec: mam.spec
           executor: otp_25_elasticsearch_cassandra
           context: mongooseim-org
           preset: elasticsearch_and_cassandra_mnesia

.github/workflows/ci.yml

@@ -14,6 +14,9 @@ env:
   AWS_DEFAULT_REGION: "${{ secrets.AWS_DEFAULT_REGION }}"
   AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
   AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+  # required for tools/gh-report-failing-testcases-to-ga4.sh script
+  GA4_API_SECRET: "${{ secrets.GA4_API_SECRET }}"
+  GA4_MEASUREMENT_API: "${{ secrets.GA4_MEASUREMENT_API }}"
 
 jobs:
   small_tests:
@@ -80,6 +83,9 @@ jobs:
       - name: upload common test results on failure
         if: ${{ failure() }}
         run: tools/gh-upload-to-s3.sh big_tests/ct_report
+      - name: upload big_tests results to GA4
+        if: github.ref_name == 'master'
+        run: tools/gh-report-failing-testcases-to-ga4.sh
 
   dynamic_domains_big_tests:
     name: dynamic domains ${{matrix.preset}} on OTP ${{matrix.otp}}
@@ -105,6 +111,9 @@ jobs:
       - name: upload common test results on failure
         if: ${{ failure() }}
         run: tools/gh-upload-to-s3.sh big_tests/ct_report
+      - name: upload big_tests results to GA4
+        if: github.ref_name == 'master'
+        run: tools/gh-report-failing-testcases-to-ga4.sh
 
   coveralls_webhook:
     needs: [big_tests, small_tests, dynamic_domains_big_tests]
@@ -163,10 +172,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pkg: [centos_7, debian_stretch]
+        pkg: [centos_7, ubuntu_xenial]
     runs-on: ubuntu-22.04
     env:
-      ESL_ERLANG_PKG_VER: "25.0.1-1"
+      ESL_ERLANG_PKG_VER: "25.0.3-1"
       pkg_PLATFORM: ${{matrix.pkg}}
     steps:
       - uses: actions/checkout@v3

big_tests/default.spec

@@ -121,15 +121,11 @@
 
 %% ct_tty_hook will log CT failures to TTY verbosely
 %% ct_mongoose_hook will:
-%% * log suite start/end events in the MongooseIM console
-%% * ensure preset value is passed to ct Config
+%% * ensure preset & mim_data_dir values are passed to ct Config
 %% * check server's purity after SUITE
 {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook,
-            ct_mim_config_hook,
-            ct_markdown_errors_hook,
-            {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]},
-            {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]}
-           ]}.
+            ct_markdown_errors_hook, ct_mongoose_log_hook]}.
 
-%% To enable printing group and case enters on server side
-%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}.
+%% since test-runner.sh can be executed with the --one-node option,
+%% log collection is enabled by default for host mim1 only.
+% {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}, {log, []}]}]}.

big_tests/dynamic_domains.spec

@@ -164,15 +164,11 @@
 
 %% ct_tty_hook will log CT failures to TTY verbosely
 %% ct_mongoose_hook will:
-%% * log suite start/end events in the MongooseIM console
-%% * ensure preset value is passed to ct Config
+%% * ensure preset & mim_data_dir values are passed to ct Config
 %% * check server's purity after SUITE
 {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook,
-            ct_mim_config_hook,
-            ct_markdown_errors_hook,
-            {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]},
-            {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]}
-           ]}.
-
-%% To enable printing group and case enters on server side
-%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}.
+            ct_markdown_errors_hook, ct_mongoose_log_hook]}.
+
+%% since test-runner.sh can be executed with the --one-node option,
+%% log collection is enabled by default for host mim1 only.
+% {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}, {log, []}]}]}.

big_tests/failure_reporting_testing.spec

@@ -0,0 +1,34 @@
+{suites, "tests", test_SUITE}.
+
+{config, ["test.config"]}.
+{logdir, "ct_report"}.
+
+%%% ct_test_hook is just a demo CT hook with tracing.
+%%% ct_test_hook should be executed just once, see
+%%% implementation of ct_test_hook:id/1 and ct_hooks
+%%% documentation:
+%%%    https://erlang.org/doc/man/ct_hooks.html
+{ct_hooks, [ct_test_hook, ct_test_hook]}.
+
+%%% this hook generates junit_report.xml file
+{ct_hooks, [cth_surefire]}.
+
+%%% this hook generates groups.summary files per suite
+%%% and an overall resulting all_groups.summary file.
+{ct_hooks, [ct_groups_summary_hook]}.
+
+%%% this hook generates /tmp/ct_summary, /tmp/ct_markdown
+%%% and /tmp/ct_markdown_truncated. these files are required
+%%% for GH and GA4 CI reporting.
+{ct_hooks, [ct_markdown_errors_hook]}.
+
+%%% this hook prints execution summary in the end of
+%%% big_tests run. however, silent_exec.sh script redirects
+%%% output into a file. set VERBOSE env variable to 1, to
+%%% tail output file.
+% {ct_hooks, [ct_tty_hook]}.
+
+%%% this hook appeands /tmp/progress file, that file
+%%% is printed asynchronously by test.sh script using
+%%% 'tail -f /tmp/progress &' command.
+% {ct_hooks, [ct_progress_hook]}.

big_tests/mam.spec

@@ -21,15 +21,11 @@
 
 %% ct_tty_hook will log CT failures to TTY verbosely
 %% ct_mongoose_hook will:
-%% * log suite start/end events in the MongooseIM console
-%% * ensure preset value is passed to ct Config
+%% * ensure preset & mim_data_dir values are passed to ct Config
 %% * check server's purity after SUITE
 {ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook,
-            ct_markdown_errors_hook,
-            ct_mim_config_hook,
-            {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]},
-            {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]}
-           ]}.
+            ct_markdown_errors_hook, ct_mongoose_log_hook]}.
 
-%% To enable printing group and case enters on server side
-%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}.
+%% since test-runner.sh can be executed with the --one-node option,
+%% log collection is enabled by default for host mim1 only.
+% {ct_hooks, [{ct_mongoose_log_hook,[{host, mim2}, {log, []}]}]}.

big_tests/run_common_test.erl

@@ -68,6 +68,12 @@ main(RawArgs) ->
                 io:format("Exiting by test cases summary: ~p~n", [ExitStatusByTestCases]),
                 init:stop(ExitStatusByTestCases);
             _ when is_integer(ExitStatusByGroups) ->
+                %% FIXME: This is incorrect assumption, it ignores results of all individual
+                %% tests cases and groups w/o 'repeat_until_all_ok' flag. So we can return
+                %% false positive result here. It's not too critical, because test.sh script
+                %% recalculates return code using summarise-ct-results utility. However, it
+                %% may result in squashing ct output, since execution of run_common_test.erl
+                %% is wrapped using silent_exec.sh tool.
                 io:format("Exiting by groups summary: ~p~n",  [ExitStatusByGroups]),
                 init:stop(ExitStatusByGroups)
         end

big_tests/src/ct_groups_summary_hook.erl

@@ -10,10 +10,16 @@
          pre_init_per_suite/3,
          post_end_per_suite/4,
          post_end_per_group/4,
+         on_tc_skip/3,
          terminate/1]).
 
+
 -include_lib("common_test/include/ct.hrl").
 
+-record(group_status, {status = ok :: ok | failed,
+                       n_failed = 0 :: pos_integer(),
+                       failed = [] :: list()}).
+
 %% @doc Return a unique id for this CTH.
 id(_Opts) ->
     "ct_results_summary_hook_001".
@@ -47,6 +53,15 @@ post_end_per_group(GroupName, Config, Return, State) ->
     ct:log("NewState: ~p", [State1]),
     {Return, State1}.
 
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing.
+on_tc_skip(_TC, Reason, State) ->
+    case Reason of
+        %% Just quit if files were not build correctly
+        {tc_user_skip, "Make failed"} -> erlang:halt(1);
+        _                             -> State
+    end.
+
 terminate(#{total_ok := OK, total_eventually_ok_tests := TotalEventuallyOK,
             total_failed := TotalFailed} = State) ->
     Content = io_lib:format("~p.~n~p.~n~p~n.",
@@ -64,41 +79,59 @@ update_group_status(GroupName, Config, State) ->
     case lists:keyfind(tc_group_properties, 1, Config) of
         false -> State;
         {tc_group_properties, Properties} ->
-            case proplists:is_defined(repeat_until_all_ok, Properties) of
-                false -> State;
-                true ->
-                    GroupNameWPath = group_name_with_path(GroupName, Config),
-                    GroupResult = ?config(tc_group_result, Config),
-                    case proplists:get_value(failed, GroupResult, []) of
-                        [] ->
-                            %% If there are no failed cases, then the only skipped cases present
-                            %% in the group result are user-skipped cases.
-                            %% In this case we do not want to fail the whole group.
-                            do_update_group_status(ok, GroupNameWPath, 0, State);
-                        Failed ->
-                            %% If there are failed cases, it doesn't matter if the skipped cases
-                            %% are user-skipped or auto-skipped (which might depend on `sequence`
-                            %% group property being enabled or not) - we fail in either case.
-                            %% TODO: Report to the respective GitHub PR
-                            ct:pal("Failed in this group: ~p", [Failed]),
-                            do_update_group_status(failed, GroupNameWPath, length(Failed), State)
-                    end
-            end
+            GroupNameWPath = group_name_with_path(GroupName, Config),
+            GroupResult = ?config(tc_group_result, Config),
+            Failed = proplists:get_value(failed, GroupResult, []),
+            Status = case Failed of
+                         [] ->
+                             %% If there are no failed cases, then the only skipped cases present
+                             %% in the group result are user-skipped cases.
+                             %% In this case we do not want to fail the whole group.
+                             ok;
+                         _ ->
+                             %% If there are failed cases, it doesn't matter if the skipped cases
+                             %% are user-skipped or auto-skipped (which might depend on `sequence`
+                             %% group property being enabled or not) - we fail in either case.
+                             failed
+                     end,
+            RepeatFlag = proplists:is_defined(repeat_until_all_ok, Properties),
+            do_update_group_status(RepeatFlag, Status, GroupNameWPath, Failed, State)
     end.
 
 
-do_update_group_status(Status, GroupNameWPath, NFailed, #{current_suite := CurrentSuite} = State) ->
+do_update_group_status(RepeatFlag, Status, GroupNameWPath, Failed,
+                       #{current_suite := CurrentSuite} = State) ->
     SuiteState = maps:get(CurrentSuite, State),
+    NewGroupStatus = #group_status{status = Status, n_failed = length(Failed),
+                                   failed = Failed},
     NewSuiteState = case maps:get(GroupNameWPath, SuiteState, undefined) of
+                        undefined when RepeatFlag =:= true ->
+                            %% repeat_until_all_ok flag works like a counter.
+                            %% it decreases with every subsequent run, and it
+                            %% is not set at all on the last attempt. so we
+                            %% want add a new group status info only if flag
+                            %% is set. otherwise, we want to update the status
+                            %% only if group is already stored.
+                            SuiteState#{GroupNameWPath => NewGroupStatus};
                         undefined ->
-                            SuiteState#{GroupNameWPath => group_status(Status, NFailed)};
-                        {_PrevStatus, {n_failed, NFailedAcc}} ->
-                            SuiteState#{GroupNameWPath => group_status(Status, NFailed + NFailedAcc)}
+                            SuiteState;
+                        PrevGroupStatus ->
+                            SuiteState#{GroupNameWPath := merge_group_status(PrevGroupStatus,
+                                                                             NewGroupStatus)}
                     end,
     State#{CurrentSuite := NewSuiteState}.
 
-group_status(Status, NFailed) ->
-    {Status, {n_failed, NFailed}}.
+merge_group_status(#group_status{failed = PrevFailed,
+                                 n_failed = PrevNFailed} = _PrevGroupStatus,
+                   #group_status{status = NewStatus, failed = NewFailed,
+                                 n_failed = NewNFailed} = _NewGroupStatus) ->
+    MergedFailed = lists:umerge(lists:usort(PrevFailed),lists:usort(NewFailed)),
+    %% we need to count the number of fails, not the number of failed test cases
+    %% (i.e. length(MergedFailed)). that's because we compare it with the number
+    %% of failed test cases reported in suite.summary files, for more details see
+    %% summarise-ct-results script.
+    MergedNFailed = PrevNFailed + NewNFailed,
+    #group_status{status = NewStatus, n_failed = MergedNFailed, failed = MergedFailed}.
 
 
 group_name_with_path(GroupName, Config) ->
@@ -131,13 +164,12 @@ write_groups_summary(Config, #{total_ok := TOK, total_failed := TFailed,
            total_failed := TFailed + Failed,
            total_eventually_ok_tests := TEvOK + FailedTests}.
 
-acc_groups_summary(_GroupName, {ok, {n_failed, NFailedTests}},
+acc_groups_summary(_GroupName, #group_status{status = ok, n_failed = NFailedTests},
                    {OkGroupsAcc, FailedGroupsAcc, EventuallyOkAcc}) ->
-    {OkGroupsAcc + 1, FailedGroupsAcc,
-     %% The group was repeated, but it eventually passed, so the tests must finally be ok.
-     EventuallyOkAcc + NFailedTests};
-acc_groups_summary(_GroupName, {__, {n_failed, _NFailedTests}},
+    %% Either the group has passed succesfully on the first run,
+    %% or it was repeated, and eventually passed, so the tests must finally be ok.
+    {OkGroupsAcc + 1, FailedGroupsAcc, EventuallyOkAcc + NFailedTests};
+acc_groups_summary(_GroupName, #group_status{status = failed, n_failed = NFailedTests},
                    {OkGroupsAcc, FailedGroupsAcc, EventuallyOkAcc}) ->
-    {OkGroupsAcc, FailedGroupsAcc + 1,
-     %% The group never succeeded, the failed tests are NOT eventually ok.
-     EventuallyOkAcc}.
+    %% The group never succeeded, the failed tests are NOT eventually ok.
+    {OkGroupsAcc, FailedGroupsAcc + 1, EventuallyOkAcc}.