diff --git a/database/scripts/get_known_issue_by_url.sql b/database/scripts/get_known_issue_by_url.sql new file mode 100644 index 0000000..78de0fe --- /dev/null +++ b/database/scripts/get_known_issue_by_url.sql @@ -0,0 +1,6 @@ +SELECT job_name, + error_name +FROM test_fail_issues +WHERE github_issue = "@param1@" +GROUP BY github_issue, + job_name; \ No newline at end of file diff --git a/database/scripts/lib/buildfarm_tools.rb b/database/scripts/lib/buildfarm_tools.rb index e0c3369..290bac9 100644 --- a/database/scripts/lib/buildfarm_tools.rb +++ b/database/scripts/lib/buildfarm_tools.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'open3' +require 'csv' module BuildfarmToolsLib class BuildfarmToolsError < RuntimeError; end @@ -15,6 +16,8 @@ class BuildfarmToolsError < RuntimeError; end FLAKY_BUILDS_DEFAULT_RANGE = '15 days' WARNING_AGE_CONSTANT = -1 + JOB_PRIORITIES = CSV.read('lib/job_priorities.csv', converters: :numeric).to_h + def self.build_regressions_today(filter_known: false) # Keys: job_name, build_number, build_datetime, failure_reason, last_section out = run_command('./sql_run.sh builds_failing_today.sql') @@ -128,13 +131,49 @@ def self.jobs_last_success_date(older_than_days: 0) out end - def self.test_regressions_known + def self.test_regressions_known(sort_by: 'priority') out = known_issues(status: 'open') out.concat known_issues(status: 'disabled') out = out.group_by { |e| e["github_issue"] }.to_a.map { |e| e[1] } + out.each do |error_list| + priority = calculate_issue_priority(error_list.first["github_issue"]) + error_list.each do |error| + error["priority"] = priority + end + end + + unless sort_by.nil? + out.sort_by! { |r| -r.first['priority'] } + end out end + def self.calculate_issue_priority(issue_link) + sql_out = run_command('./sql_run.sh get_known_issue_by_url.sql', args: [issue_link]) + errors = sql_out.map {|e| e['error_name']}.uniq + jobs = sql_out.map {|e| e['job_name']}.uniq + + error_score_jobs = {} + + errors.each do |e| + jobs.each do |job| + flaky_result = run_command('./sql_run.sh calculate_flakiness_jobs.sql', args: [e, FLAKY_BUILDS_DEFAULT_RANGE, job]) + next if flaky_result.empty? + # This is not guaranteed to be 'not consistent', we need to re-check if the last 3 builds were failing because of this + flaky_ratio = flaky_result.first['failure_percentage'].to_f/100.0 + + job_priority = JOB_PRIORITIES[job] + job_priority = job_priority*1.5 if flaky_ratio == 1 + + error_score_jobs[job] = [] if error_score_jobs[job].nil? + error_score_jobs[job] << (job_priority*flaky_ratio) + end + end + + # Get only maximum score for each job + error_score_jobs.each_value.map {|e| e.max}.sum.round(3) + end + def self.run_command(cmd, args: [], keys: []) cmd += " '#{args.shift}'" until args.empty? begin diff --git a/database/scripts/lib/job_priorities.csv b/database/scripts/lib/job_priorities.csv new file mode 100644 index 0000000..5258ea4 --- /dev/null +++ b/database/scripts/lib/job_priorities.csv @@ -0,0 +1,335 @@ +nightly_linux_debug,1 +nightly_linux_release,1 +nightly_linux_repeated,1 +packaging_linux,1 +nightly_win_deb,1 +nightly_win_rel,1 +nightly_win_rep,1 +packaging_windows,1 +nightly_linux-aarch64_debug,1 +nightly_linux-aarch64_release,1 +nightly_linux-aarch64_repeated,1 +packaging_linux-aarch64,1 +Rci__nightly-debug_ubuntu_noble_amd64,1 +Rci__nightly-release_ubuntu_noble_amd64,1 +Jci__nightly-debug_ubuntu_noble_amd64,0.95 +Jci__nightly-release_ubuntu_noble_amd64,0.95 +Ici__nightly-debug_ubuntu_jammy_amd64,0.9 +Ici__nightly-release_ubuntu_jammy_amd64,0.9 +Hci__nightly-debug_ubuntu_jammy_amd64,0.85 +Hci__nightly-release_ubuntu_jammy_amd64,0.85 +Rci__nightly-fastrtps_ubuntu_noble_amd64,0.85 +Rci__nightly-cyclonedds_ubuntu_noble_amd64,0.85 +Rci__nightly-connext_ubuntu_noble_amd64,0.85 +nightly_linux_coverage,0.85 +Jci__nightly-fastrtps_ubuntu_noble_amd64,0.8 +Jci__nightly-cyclonedds_ubuntu_noble_amd64,0.8 +Jci__nightly-connext_ubuntu_noble_amd64,0.8 +nightly_linux_jazzy_coverage,0.8 +Ici__nightly-fastrtps_ubuntu_jammy_amd64,0.75 +Ici__nightly-cyclonedds_ubuntu_jammy_amd64,0.75 +Ici__nightly-connext_ubuntu_jammy_amd64,0.75 +nightly_linux_iron_coverage,0.75 +Hci__nightly-fastrtps_ubuntu_jammy_amd64,0.7 +Hci__nightly-cyclonedds_ubuntu_jammy_amd64,0.7 +Hci__nightly-connext_ubuntu_jammy_amd64,0.7 +nightly_linux_humble_coverage,0.7 +nightly_linux-rhel_debug,0.5 +nightly_linux-rhel_release,0.5 +nightly_linux-rhel_repeated,0.5 +packaging_linux-rhel,0.5 +Hci__nightly-fastrtps-dynamic_ubuntu_jammy_amd64,0.25 +Hci__nightly-performance_ubuntu_jammy_amd64,0.25 +Hci__benchmark_ubuntu_jammy_amd64,0.25 +Ici__nightly-fastrtps-dynamic_ubuntu_jammy_amd64,0.25 +Ici__nightly-performance_ubuntu_jammy_amd64,0.25 +Ici__benchmark_ubuntu_jammy_amd64,0.25 +Jci__nightly-fastrtps-dynamic_ubuntu_noble_amd64,0.25 +Jci__nightly-performance_ubuntu_noble_amd64,0.25 +Jci__benchmark_ubuntu_noble_amd64,0.25 +Rci__nightly-fastrtps-dynamic_ubuntu_noble_amd64,0.25 +Rci__nightly-performance_ubuntu_noble_amd64,0.25 +Rci__benchmark_ubuntu_noble_amd64,0.25 +gz-cmake4-debbuilder,1 +gz-cmake4-debbuilder,1 +gz-common6-debbuilder,1 +gz-common6-debbuilder,1 +gz-fuel-tools10-debbuilder,1 +gz-fuel-tools10-debbuilder,1 +gz-gui9-debbuilder,1 +gz-gui9-debbuilder,1 +gz-launch8-debbuilder,1 +gz-launch8-debbuilder,1 +gz-math8-debbuilder,1 +gz-math8-debbuilder,1 +gz-msgs11-debbuilder,1 +gz-msgs11-debbuilder,1 +gz-physics8-debbuilder,1 +gz-physics8-debbuilder,1 +gz-plugin3-debbuilder,1 +gz-plugin3-debbuilder,1 +gz-rendering9-debbuilder,1 +gz-rendering9-debbuilder,1 +gz-sensors9-debbuilder,1 +gz-sensors9-debbuilder,1 +gz-sim9-debbuilder,1 +gz-sim9-debbuilder,1 +gz-tools2-debbuilder,1 +gz-tools2-debbuilder,1 +gz-transport14-debbuilder,1 +gz-transport14-debbuilder,1 +gz-utils3-debbuilder,1 +gz-utils3-debbuilder,1 +gz_cmake-4-win,0.9 +gz_cmake-ci-gz-cmake4-homebrew-amd64,0.9 +gz_cmake-ci-gz-cmake4-noble-amd64,1 +gz_common-6-win,0.9 +gz_common-ci-gz-common6-homebrew-amd64,0.9 +gz_common-ci-gz-common6-noble-amd64,1 +gz_fuel_tools-10-win,0.9 +gz_fuel_tools-ci-gz-fuel-tools10-homebrew-amd64,0.9 +gz_fuel_tools-ci-gz-fuel-tools10-noble-amd64,1 +gz_gui-9-win,0.9 +gz_gui-ci-gz-gui9-homebrew-amd64,0.9 +gz_gui-ci-gz-gui9-noble-amd64,1 +gz_launch-8-win,0.9 +gz_launch-ci-gz-launch8-homebrew-amd64,0.9 +gz_launch-ci-gz-launch8-noble-amd64,1 +gz_math-8-win,0.9 +gz_math-ci-gz-math8-homebrew-amd64,0.9 +gz_math-ci-gz-math8-noble-amd64,1 +gz_msgs-11-win,0.9 +gz_msgs-ci-gz-msgs11-homebrew-amd64,0.9 +gz_msgs-ci-gz-msgs11-noble-amd64,1 +gz_physics-8-win,0.9 +gz_physics-ci-gz-physics8-homebrew-amd64,0.9 +gz_physics-ci-gz-physics8-noble-amd64,1 +gz_plugin-3-win,0.9 +gz_plugin-ci-gz-plugin3-homebrew-amd64,0.9 +gz_plugin-ci-gz-plugin3-noble-amd64,1 +gz_rendering-9-win,0.9 +gz_rendering-ci-gz-rendering9-homebrew-amd64,0.9 +gz_rendering-ci-gz-rendering9-noble-amd64,1 +gz_sensors-9-win,0.9 +gz_sensors-ci-gz-sensors9-homebrew-amd64,0.9 +gz_sensors-ci-gz-sensors9-noble-amd64,1 +gz_sim-9-win,0.9 +gz_sim-ci-gz-sim9-homebrew-amd64,0.9 +gz_sim-ci-gz-sim9-noble-amd64,1 +gz_tools-2-win,0.9 +gz_tools-ci-gz-tools2-homebrew-amd64,0.9 +gz_tools-ci-gz-tools2-noble-amd64,1 +gz_transport-14-win,0.9 +gz_transport-ci-gz-transport14-homebrew-amd64,0.9 +gz_transport-ci-gz-transport14-noble-amd64,1 +gz_utils-3-win,0.9 +gz_utils-ci-gz-utils3-homebrew-amd64,0.9 +gz_utils-ci-gz-utils3-noble-amd64,1 +sdformat-ci-sdf15-homebrew-amd64,0.9 +sdformat-ci-sdf15-noble-amd64,1 +sdformat-sdf15-win,0.9 +sdformat15-debbuilder,1 +sdformat15-debbuilder,1 +gz_cmake-3-win,0.85 +gz_cmake-ci-gz-cmake3-homebrew-amd64,0.85 +gz_cmake-ci-gz-cmake3-jammy-amd64,0.95 +gz_cmake-ci-gz-cmake3-noble-amd64,0.95 +gz_common-5-win,0.85 +gz_common-ci-gz-common5-homebrew-amd64,0.85 +gz_common-ci-gz-common5-jammy-amd64,0.95 +gz_common-ci-gz-common5-noble-amd64,0.95 +gz_fuel_tools-9-win,0.85 +gz_fuel_tools-ci-gz-fuel-tools9-homebrew-amd64,0.85 +gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64,0.95 +gz_fuel_tools-ci-gz-fuel-tools9-noble-amd64,0.95 +gz_gui-8-win,0.85 +gz_gui-ci-gz-gui8-homebrew-amd64,0.85 +gz_gui-ci-gz-gui8-jammy-amd64,0.95 +gz_gui-ci-gz-gui8-noble-amd64,0.95 +gz_launch-7-win,0.85 +gz_launch-ci-gz-launch7-homebrew-amd64,0.85 +gz_launch-ci-gz-launch7-jammy-amd64,0.95 +gz_launch-ci-gz-launch7-noble-amd64,0.95 +gz_math-7-win,0.85 +gz_math-ci-gz-math7-homebrew-amd64,0.85 +gz_math-ci-gz-math7-jammy-amd64,0.95 +gz_math-ci-gz-math7-noble-amd64,0.95 +gz_msgs-10-win,0.85 +gz_msgs-ci-gz-msgs10-homebrew-amd64,0.85 +gz_msgs-ci-gz-msgs10-jammy-amd64,0.95 +gz_msgs-ci-gz-msgs10-noble-amd64,0.95 +gz_physics-7-win,0.85 +gz_physics-ci-gz-physics7-homebrew-amd64,0.85 +gz_physics-ci-gz-physics7-jammy-amd64,0.95 +gz_physics-ci-gz-physics7-noble-amd64,0.95 +gz_plugin-2-win,0.85 +gz_plugin-ci-gz-plugin2-homebrew-amd64,0.85 +gz_plugin-ci-gz-plugin2-jammy-amd64,0.95 +gz_plugin-ci-gz-plugin2-noble-amd64,0.95 +gz_rendering-8-win,0.85 +gz_rendering-ci-gz-rendering8-homebrew-amd64,0.85 +gz_rendering-ci-gz-rendering8-jammy-amd64,0.95 +gz_rendering-ci-gz-rendering8-noble-amd64,0.95 +gz_sensors-8-win,0.85 +gz_sensors-ci-gz-sensors8-homebrew-amd64,0.85 +gz_sensors-ci-gz-sensors8-jammy-amd64,0.95 +gz_sensors-ci-gz-sensors8-noble-amd64,0.95 +gz_sim-8-win,0.85 +gz_sim-ci-gz-sim8-homebrew-amd64,0.85 +gz_sim-ci-gz-sim8-jammy-amd64,0.95 +gz_sim-ci-gz-sim8-noble-amd64,0.95 +gz_tools-2-win,0.85 +gz_tools-ci-gz-tools2-homebrew-amd64,0.85 +gz_tools-ci-gz-tools2-jammy-amd64,0.95 +gz_tools-ci-gz-tools2-noble-amd64,0.95 +gz_transport-13-win,0.85 +gz_transport-ci-gz-transport13-homebrew-amd64,0.85 +gz_transport-ci-gz-transport13-jammy-amd64,0.95 +gz_transport-ci-gz-transport13-noble-amd64,0.95 +gz_utils-2-win,0.85 +gz_utils-ci-gz-utils2-homebrew-amd64,0.85 +gz_utils-ci-gz-utils2-jammy-amd64,0.95 +gz_utils-ci-gz-utils2-noble-amd64,0.95 +sdformat-ci-sdf14-homebrew-amd64,0.85 +sdformat-ci-sdf14-jammy-amd64,0.95 +sdformat-ci-sdf14-noble-amd64,0.95 +sdformat-sdf14-win,0.85 +gz_cmake-3-win,0.8 +gz_cmake-ci-gz-cmake3-focal-amd64,0.9 +gz_cmake-ci-gz-cmake3-homebrew-amd64,0.8 +gz_common-5-win,0.8 +gz_common-ci-gz-common5-focal-amd64,0.9 +gz_common-ci-gz-common5-homebrew-amd64,0.8 +gz_fuel_tools-8-win,0.8 +gz_fuel_tools-ci-gz-fuel-tools8-focal-amd64,0.9 +gz_fuel_tools-ci-gz-fuel-tools8-homebrew-amd64,0.8 +gz_gui-7-win,0.8 +gz_gui-ci-gz-gui7-focal-amd64,0.9 +gz_gui-ci-gz-gui7-homebrew-amd64,0.8 +gz_launch-6-win,0.8 +gz_launch-ci-gz-launch6-focal-amd64,0.9 +gz_launch-ci-gz-launch6-homebrew-amd64,0.8 +gz_math-7-win,0.8 +gz_math-ci-gz-math7-focal-amd64,0.9 +gz_math-ci-gz-math7-homebrew-amd64,0.8 +gz_msgs-9-win,0.8 +gz_msgs-ci-gz-msgs9-focal-amd64,0.9 +gz_msgs-ci-gz-msgs9-homebrew-amd64,0.8 +gz_physics-6-win,0.8 +gz_physics-ci-gz-physics6-focal-amd64,0.9 +gz_physics-ci-gz-physics6-homebrew-amd64,0.8 +gz_plugin-2-win,0.8 +gz_plugin-ci-gz-plugin2-focal-amd64,0.9 +gz_plugin-ci-gz-plugin2-homebrew-amd64,0.8 +gz_rendering-7-win,0.8 +gz_rendering-ci-gz-rendering7-focal-amd64,0.9 +gz_rendering-ci-gz-rendering7-homebrew-amd64,0.8 +gz_sensors-7-win,0.8 +gz_sensors-ci-gz-sensors7-focal-amd64,0.9 +gz_sensors-ci-gz-sensors7-homebrew-amd64,0.8 +gz_sim-7-win,0.8 +gz_sim-ci-gz-sim7-focal-amd64,0.9 +gz_sim-ci-gz-sim7-homebrew-amd64,0.8 +gz_tools-2-win,0.8 +gz_tools-ci-gz-tools2-focal-amd64,0.9 +gz_tools-ci-gz-tools2-homebrew-amd64,0.8 +gz_transport-12-win,0.8 +gz_transport-ci-gz-transport12-focal-amd64,0.9 +gz_transport-ci-gz-transport12-homebrew-amd64,0.8 +gz_utils-2-win,0.8 +gz_utils-ci-gz-utils2-focal-amd64,0.9 +gz_utils-ci-gz-utils2-homebrew-amd64,0.8 +sdformat-ci-sdf13-focal-amd64,0.9 +sdformat-ci-sdf13-homebrew-amd64,0.8 +sdformat-sdf13-win,0.8 +gz_cmake-ci-ign-cmake2-focal-amd64,0.85 +gz_cmake-ci-ign-cmake2-homebrew-amd64,0.75 +gz_cmake-ign-cmake2-win,0.75 +gz_common-ci-ign-common4-focal-amd64,0.85 +gz_common-ci-ign-common4-homebrew-amd64,0.75 +gz_common-ign-common4-win,0.75 +gz_fuel_tools-ci-ign-fuel-tools7-focal-amd64,0.85 +gz_fuel_tools-ci-ign-fuel-tools7-homebrew-amd64,0.75 +gz_fuel_tools-ign-fuel-tools7-win,0.75 +gz_gui-ci-ign-gui6-focal-amd64,0.85 +gz_gui-ci-ign-gui6-homebrew-amd64,0.75 +gz_gui-ign-gui6-win,0.75 +gz_launch-ci-ign-launch5-focal-amd64,0.85 +gz_launch-ci-ign-launch5-homebrew-amd64,0.75 +gz_launch-ign-launch5-win,0.75 +gz_math-ci-ign-math6-focal-amd64,0.85 +gz_math-ci-ign-math6-homebrew-amd64,0.75 +gz_math-ign-math6-win,0.75 +gz_msgs-ci-ign-msgs8-focal-amd64,0.85 +gz_msgs-ci-ign-msgs8-homebrew-amd64,0.75 +gz_msgs-ign-msgs8-win,0.75 +gz_physics-ci-ign-physics5-focal-amd64,0.85 +gz_physics-ci-ign-physics5-homebrew-amd64,0.75 +gz_physics-ign-physics5-win,0.75 +gz_plugin-ci-ign-plugin1-focal-amd64,0.85 +gz_plugin-ci-ign-plugin1-homebrew-amd64,0.75 +gz_plugin-ign-plugin1-win,0.75 +gz_rendering-ci-ign-rendering6-focal-amd64,0.85 +gz_rendering-ci-ign-rendering6-homebrew-amd64,0.75 +gz_rendering-ign-rendering6-win,0.75 +gz_sensors-ci-ign-sensors6-focal-amd64,0.85 +gz_sensors-ci-ign-sensors6-homebrew-amd64,0.75 +gz_sensors-ign-sensors6-win,0.75 +gz_sim-ci-ign-gazebo6-focal-amd64,0.85 +gz_sim-ci-ign-gazebo6-homebrew-amd64,0.75 +gz_sim-ign-gazebo6-win,0.75 +gz_tools-ci-ign-tools1-focal-amd64,0.85 +gz_tools-ci-ign-tools1-homebrew-amd64,0.75 +gz_tools-ign-tools1-win,0.75 +gz_transport-ci-ign-transport11-focal-amd64,0.85 +gz_transport-ci-ign-transport11-homebrew-amd64,0.75 +gz_transport-ign-transport11-win,0.75 +gz_utils-ci-ign-utils1-focal-amd64,0.85 +gz_utils-ci-ign-utils1-homebrew-amd64,0.75 +gz_utils-ign-utils1-win,0.75 +sdformat-ci-sdf12-focal-amd64,0.85 +sdformat-ci-sdf12-homebrew-amd64,0.75 +sdformat-sdf12-win,0.75 +gz_cmake-ci-ign-cmake2-focal-amd64,0.8 +gz_cmake-ci-ign-cmake2-homebrew-amd64,0.7 +gz_cmake-ign-cmake2-win,0.7 +gz_common-ci-ign-common3-focal-amd64,0.8 +gz_common-ci-ign-common3-homebrew-amd64,0.7 +gz_common-ign-common3-win,0.7 +gz_fuel_tools-ci-ign-fuel-tools4-focal-amd64,0.8 +gz_fuel_tools-ci-ign-fuel-tools4-homebrew-amd64,0.7 +gz_fuel_tools-ign-fuel-tools4-win,0.7 +gz_gui-ci-ign-gui3-focal-amd64,0.8 +gz_gui-ci-ign-gui3-homebrew-amd64,0.7 +gz_gui-ign-gui3-win,0.7 +gz_launch-ci-ign-launch2-focal-amd64,0.8 +gz_launch-ci-ign-launch2-homebrew-amd64,0.7 +gz_math-ci-ign-math6-focal-amd64,0.8 +gz_math-ci-ign-math6-homebrew-amd64,0.7 +gz_math-ign-math6-win,0.7 +gz_msgs-ci-ign-msgs5-focal-amd64,0.8 +gz_msgs-ci-ign-msgs5-homebrew-amd64,0.7 +gz_msgs-ign-msgs5-win,0.7 +gz_physics-ci-ign-physics2-focal-amd64,0.8 +gz_physics-ci-ign-physics2-homebrew-amd64,0.7 +gz_physics-ign-physics2-win,0.7 +gz_plugin-ci-ign-plugin1-focal-amd64,0.8 +gz_plugin-ci-ign-plugin1-homebrew-amd64,0.7 +gz_plugin-ign-plugin1-win,0.7 +gz_rendering-ci-ign-rendering3-focal-amd64,0.8 +gz_rendering-ci-ign-rendering3-homebrew-amd64,0.7 +gz_rendering-ign-rendering3-win,0.7 +gz_sensors-ci-ign-sensors3-focal-amd64,0.8 +gz_sensors-ci-ign-sensors3-homebrew-amd64,0.7 +gz_sensors-ign-sensors3-win,0.7 +gz_sim-ci-ign-gazebo3-focal-amd64,0.8 +gz_sim-ci-ign-gazebo3-homebrew-amd64,0.7 +gz_tools-ci-ign-tools1-focal-amd64,0.8 +gz_tools-ci-ign-tools1-homebrew-amd64,0.7 +gz_tools-ign-tools1-win,0.7 +gz_transport-ci-ign-transport8-focal-amd64,0.8 +gz_transport-ci-ign-transport8-homebrew-amd64,0.7 +gz_transport-ign-transport8-win,0.7 +sdformat-ci-sdf9-focal-amd64,0.8 +sdformat-ci-sdf9-homebrew-amd64,0.7 +sdformat-sdf9-win,0.7 diff --git a/database/scripts/lib/report_formatter.rb b/database/scripts/lib/report_formatter.rb index 6a7b41b..16e0f6b 100644 --- a/database/scripts/lib/report_formatter.rb +++ b/database/scripts/lib/report_formatter.rb @@ -154,7 +154,7 @@ def self.test_regressions_all(test_regressions_array) def self.test_regressions_known(issue_array) # Create a table for each project: {'ros' => {'open' => , 'disabled' =>
}, 'gazebo' => {'open' =>
, 'disabled' =>
}} - table_template = "| Issue | Jobs Name | Errors Name |\n| -- | -- | -- |\n" + table_template = "| Issue | Priority | Jobs Name | Errors Name |\n| -- | -- | -- | -- |\n" table_by_status = {'open' => table_template, 'disabled' => table_template} tables = Hash[TRACKED_PROJECTS.zip([table_by_status, table_by_status.clone])] @@ -169,7 +169,7 @@ def self.test_regressions_known(issue_array) # Add issue report data to it's respective project table project = iss_report.first['project'] - tables[project][iss_report.first['status'].downcase] += "| `#{iss_report[0]['github_issue']}` | #{jobs_str} | #{errors_str} |\n" + tables[project][iss_report.first['status'].downcase] += "| `#{iss_report.first['github_issue']}` | #{iss_report.first['priority']} | #{jobs_str} | #{errors_str} |\n" end out = ""