From 866fee8a4cb92b5115ef7ea878b8f2f395d695ea Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 24 Oct 2023 09:22:16 +1000 Subject: [PATCH 1/4] [#135] Add cache consistency check --- classes/check/cachecheck.php | 134 +++++++++++++++++++++++++++++++++++ classes/task/cachecheck.php | 48 +++++++++++++ db/install.php | 35 +++++++++ db/tasks.php | 36 ++++++++++ db/upgrade.php | 41 +++++++++++ lang/en/tool_heartbeat.php | 6 ++ lib.php | 10 +++ version.php | 4 +- 8 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 classes/check/cachecheck.php create mode 100644 classes/task/cachecheck.php create mode 100644 db/install.php create mode 100644 db/tasks.php create mode 100644 db/upgrade.php diff --git a/classes/check/cachecheck.php b/classes/check/cachecheck.php new file mode 100644 index 0000000..c871fd6 --- /dev/null +++ b/classes/check/cachecheck.php @@ -0,0 +1,134 @@ +. + +namespace tool_heartbeat\check; +use core\check\check; +use core\check\result; + +/** + * Cache check class + * + * This detects some split brain cache setups + * + * @package tool_heartbeat + * @author Brendan Heywood + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cachecheck extends check { + + /** + * Get Result. + * + * @return result + */ + public function get_result() : result { + $results = $this->check('web'); + $results += $this->check('cron'); + + list($status, $summary) = $this->build_result($results); + + $details = ''; + + if ($status != result::OK) { + $details .= get_string('checkcachedetails', 'tool_heartbeat'); + } + + $details .= ''; + + foreach ($results as $key => $value) { + $details .= \html_writer::start_tag('tr'); + $details .= \html_writer::tag('td', $key); + $details .= \html_writer::tag('td', $value); + + // Use DATE_RSS to show seconds, as well as timezone. + $details .= \html_writer::tag('td', date(DATE_RSS, $value)); + $details .= \html_writer::end_tag('tr'); + } + $details .= '
'; + return new result($status, $summary, $details); + } + + /** + * Reads the results and buils a check API result. + * @param array $results from check() function. + * @return array of [result status, summary string] + */ + private function build_result(array $results): array { + // Nothing set for web API. + if (empty($results['webapi'])) { + return [result::CRITICAL, get_string('checkcachewebmissing', 'tool_heartbeat')]; + } + + // Nothing set for cron API. + if (empty($results['cronapi'])) { + return [result::CRITICAL, get_string('checkcachecronmissing', 'tool_heartbeat')]; + } + + // Check for split cron cache/db, web cache/db, and all of them together. + $cronsplit = $results['cronapi'] != $results['crondb']; + $websplit = $results['webapi'] != $results['webdb']; + + if ($cronsplit || $websplit) { + $splits = [ + 'cron' => $cronsplit, + 'web' => $websplit, + ]; + $splits = implode(",", array_keys(array_filter($splits))); + + return [result::CRITICAL, get_string('checkcacheerrorsplit', 'tool_heartbeat', $splits)]; + } + + // Else OK. + return [result::OK, get_string('checkcachenotsplit', 'tool_heartbeat')]; + } + + /** + * Get the ping values from the cache and db to compare + * @param string $type type of check (e.g. web, cron) + */ + public function check($type) { + global $DB; + + $return = []; + $key = "checkcache{$type}ping"; + + // Read from cache (e.g. get_config uses cache). + $return[$type . 'api'] = get_config('tool_heartbeat', $key); + + // Read directly from database. + $return[$type . 'db'] = $DB->get_field('config_plugins', 'value', [ + 'plugin' => 'tool_heartbeat', + 'name' => $key, + ]); + return $return; + } + + /** + * Sets a timestamp in config from web or cron + * @param string $type type of check (e.g. web, cron) + */ + public static function ping($type) { + $key = "checkcache{$type}ping"; + $current = get_config('tool_heartbeat', $key); + + // Only update if the currently cached time is very old. + if ($current < (time() - DAYSECS)) { + mtrace("\nHEARTBEAT doing {$type} ping {$current}\n", DEBUG_DEVELOPER); + set_config($key, time(), 'tool_heartbeat'); + } + } +} diff --git a/classes/task/cachecheck.php b/classes/task/cachecheck.php new file mode 100644 index 0000000..59b2f69 --- /dev/null +++ b/classes/task/cachecheck.php @@ -0,0 +1,48 @@ +. + + +namespace tool_heartbeat\task; + +/** + * Scheduled task to ping the cache from CRON. + * + * @package tool_heartbeat + * @author Brendan Heywood + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cachecheck extends \core\task\scheduled_task { + + /** + * Get task name + */ + public function get_name() { + return get_string('checkcachecheck', 'tool_heartbeat'); + } + + /** + * Execute task + */ + public function execute() { + if (class_exists('\core\check\manager')) { + \tool_heartbeat\check\cachecheck::ping('cron'); + } + } + +} + + diff --git a/db/install.php b/db/install.php new file mode 100644 index 0000000..a8fe591 --- /dev/null +++ b/db/install.php @@ -0,0 +1,35 @@ +. +/** + * Cache split check. + * + * @package tool_heartbeat + * @author Brendan Heywood + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Install + */ +function xmldb_tool_heartbeat_install() { + // If there are issues with split caches they need to be exposed + // after some time for them to diverge. + if (class_exists('\core\check\manager')) { + \tool_heartbeat\check\cachecheck::ping('web'); + \tool_heartbeat\check\cachecheck::ping('cron'); + } +} diff --git a/db/tasks.php b/db/tasks.php new file mode 100644 index 0000000..f1bed20 --- /dev/null +++ b/db/tasks.php @@ -0,0 +1,36 @@ +. +/** + * Tool heartbeat + * + * @author Brendan Heywood + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$tasks = [ + [ + 'classname' => 'tool_heartbeat\task\cachecheck', + 'minute' => '*', + 'hour' => '*', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*', + ], +]; + diff --git a/db/upgrade.php b/db/upgrade.php new file mode 100644 index 0000000..7a39d4b --- /dev/null +++ b/db/upgrade.php @@ -0,0 +1,41 @@ +. +/** + * DB upgrade script. + * + * @package tool_heartbeat + * @author Matthew Hilton + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Upgrade + * @param int $oldversion + */ +function xmldb_tool_heartbeat_upgrade($oldversion) { + if ($oldversion < 2023102400) { + // If there are issues with split caches they need to be exposed + // after some time for them to diverge. + if (class_exists('\core\check\manager')) { + \tool_heartbeat\check\cachecheck::ping('web'); + \tool_heartbeat\check\cachecheck::ping('cron'); + } + upgrade_main_savepoint(true, 2023102400); + } + + return true; +} diff --git a/lang/en/tool_heartbeat.php b/lang/en/tool_heartbeat.php index f50c665..637a027 100644 --- a/lang/en/tool_heartbeat.php +++ b/lang/en/tool_heartbeat.php @@ -58,6 +58,12 @@ $string['configuredauths'] = 'Check auth methods'; $string['configuredauthsdesc'] = 'Auth methods to check are enabled in the Check API. A warning will be emitted if they are not enabled.'; $string['checkauthcheck'] = 'Authentication methods'; +$string['checkcachecheck'] = 'Cache consistency check'; +$string['checkcachecronmissing'] = 'The cron cache check has not succeeded yet or is missing'; +$string['checkcachedetails'] = 'A split brain cache was detected. The value stored in the database table config_plugins was not the same as the cached value returned from get_config. If you purge the cache and this check passes and then fails again after a few hours then that strongly suggests a cache misconfiguration.'; +$string['checkcacheerrorsplit'] = 'The caches are not consistent: {$a}'; +$string['checkcachenotsplit'] = 'Caches appear consistent between web and cron'; +$string['checkcachewebmissing'] = 'The web cache check has not succeeded yet'; $string['checkdnscheck'] = 'DNS check'; $string['checkrangerequestcheck'] = 'Range requests check'; $string['checkrangerequestok'] = 'Range requests are working, 206 response with only 10 bytes of data'; diff --git a/lib.php b/lib.php index f248803..6a4f45d 100644 --- a/lib.php +++ b/lib.php @@ -21,6 +21,15 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +/** + * Runs before HTTP headers. Used to ping the cachecheck. + */ +function tool_heartbeat_before_http_headers() { + if (class_exists('\core\check\manager')) { + \tool_heartbeat\check\cachecheck::ping('web'); + } +} + /** * Status checks. * @@ -29,6 +38,7 @@ function tool_heartbeat_status_checks() { return [ new \tool_heartbeat\check\authcheck(), + new \tool_heartbeat\check\cachecheck(), new \tool_heartbeat\check\logstorecheck(), new \tool_heartbeat\check\tasklatencycheck(), ]; diff --git a/version.php b/version.php index 68f9f9c..41c4544 100644 --- a/version.php +++ b/version.php @@ -24,8 +24,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023101100; -$plugin->release = 2023101100; // Match release exactly to version. +$plugin->version = 2023102400; +$plugin->release = 2023102400; // Match release exactly to version. $plugin->requires = 2012120311; // Deep support going back to 2.4. $plugin->supported = [24, 401]; $plugin->component = 'tool_heartbeat'; From ed467543ea5f4f897e4e2fc3dd7240553e33e3d6 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 26 Oct 2023 12:57:55 +1000 Subject: [PATCH 2/4] docs: update branches table in README --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a8971b5..9d8005e 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ # A heartbeat test page for Moodle - [A heartbeat test page for Moodle](#a-heartbeat-test-page-for-moodle) +- [Branches](#branches) - [What is this?](#what-is-this) - [Front end health](#front-end-health) - [Application health](#application-health) - [Failed login detection](#failed-login-detection) -- [Branches](#branches) - [Installation](#installation) - [Configuration](#configuration) - [Testing](#testing) @@ -20,6 +20,19 @@ NOTE: In an ideal world this plugin should be redundant and most of it's functio https://tracker.moodle.org/browse/MDL-47271 +# Branches + +| Branch | Moodle version | PHP Version | +| ------------------ | -------------- | ----------- | +| master | Moodle 2.7 + | Php 5.4.4+ | +| MOODLE_39_STABLE | Moodle 3.9 + | Php 7.2+ | + +The master branch is always stable and should retain very deep support for old Totara's and Moodle's back to Moodle 2.7 + +For this reason we will continue to support php5 for some time. + +The MOODLE_39_STABLE branch uses the [Check API](https://moodledev.io/docs/apis/subsystems/check) exclusively. + ## Front end health @@ -109,18 +122,6 @@ The various thresholds can be configured with query params or cli args see this php loginchecker.php -h ``` -# Branches - -| Branch | Version | -| ----------- | ----------- | -| master | Moodle 2.7 + | -| MOODLE_39_STABLE | Moodle 3.9 + | - -The master branch is always stable and should retain very deep support for old Totara's and Moodle's back to Moodle 2.7 - -For this reason we will continue to support php5 for some time. - -The MOODLE_39_STABLE branch uses the [Check API](https://moodledev.io/docs/apis/subsystems/check) exclusively. # Installation From 9570d848d80a7d751025a7e6cf12f5afefe0bf92 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 26 Oct 2023 13:01:29 +1000 Subject: [PATCH 3/4] [#135] Reduce number of times task is run --- db/tasks.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/tasks.php b/db/tasks.php index f1bed20..1dc9441 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -27,7 +27,7 @@ [ 'classname' => 'tool_heartbeat\task\cachecheck', 'minute' => '*', - 'hour' => '*', + 'hour' => '*/8', 'day' => '*', 'dayofweek' => '*', 'month' => '*', From 92e30feca182e264d1e45fb790f46d90570c8f76 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 26 Oct 2023 13:30:25 +1000 Subject: [PATCH 4/4] docs: update master branch moodle version range --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9d8005e..596cd5b 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,16 @@ https://tracker.moodle.org/browse/MDL-47271 # Branches -| Branch | Moodle version | PHP Version | -| ------------------ | -------------- | ----------- | -| master | Moodle 2.7 + | Php 5.4.4+ | -| MOODLE_39_STABLE | Moodle 3.9 + | Php 7.2+ | +| Branch | Moodle version | PHP Version | +| ------------------ | ----------------- | ----------- | +| master | Moodle 2.7 - 4.1 | Php 5.4.4+ | +| MOODLE_39_STABLE | Moodle 3.9 + | Php 7.2+ | -The master branch is always stable and should retain very deep support for old Totara's and Moodle's back to Moodle 2.7 +The master branch retains very deep support for old Totara's and Moodle's back to Moodle 2.7. -For this reason we will continue to support php5 for some time. - -The MOODLE_39_STABLE branch uses the [Check API](https://moodledev.io/docs/apis/subsystems/check) exclusively. +For any site using Moodle 3.9 or later, it is recommended to use the MOODLE_39_STABLE branch. +The MOODLE_39_STABLE branch uses the [Check API](https://moodledev.io/docs/apis/subsystems/check) exclusively, which simplifies the code massively. ## Front end health