diff --git a/classes/check/cachecheck.php b/classes/check/cachecheck.php new file mode 100644 index 0000000..b6221ce --- /dev/null +++ b/classes/check/cachecheck.php @@ -0,0 +1,134 @@ +. + +namespace tool_heartbeat\check; +use core\check\check; +use core\check\result; + +/** + * Cache check class + * + * This detects some split brain cache setups + * + * @package tool_heartbeat + * @author Brendan Heywood + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cachecheck extends check { + + /** + * Get Result. + * + * @return result + */ + public function get_result() : result { + $results = $this->check('web'); + $results += $this->check('cron'); + + list($status, $summary) = $this->build_result($results); + + $details = ''; + + if ($status != result::OK) { + $details .= get_string('checkcachedetails', 'tool_heartbeat'); + } + + $details .= ''; + + foreach ($results as $key => $value) { + $details .= \html_writer::start_tag('tr'); + $details .= \html_writer::tag('td', $key); + $details .= \html_writer::tag('td', $value); + + // Use DATE_RSS to show seconds, as well as timezone. + $details .= \html_writer::tag('td', date(DATE_RSS, $value)); + $details .= \html_writer::end_tag('tr'); + } + $details .= '
'; + return new result($status, $summary, $details); + } + + /** + * Reads the results and buils a check API result. + * @param array $results from check() function. + * @return array of [result status, summary string] + */ + private function build_result(array $results): array { + // Nothing set for web API. + if (empty($results['webapi'])) { + return [result::CRITICAL, get_string('checkcachewebmissing', 'tool_heartbeat')]; + } + + // Nothing set for cron API. + if (empty($results['cronapi'])) { + return [result::CRITICAL, get_string('checkcachecronmissing', 'tool_heartbeat')]; + } + + // Check for split cron cache/db, web cache/db, and all of them together. + $cronsplit = $results['cronapi'] != $results['crondb']; + $websplit = $results['webapi'] != $results['webdb']; + + if ($cronsplit || $websplit) { + $splits = [ + 'cron' => $cronsplit, + 'web' => $websplit, + ]; + $splits = implode(",", array_keys(array_filter($splits))); + + return [result::CRITICAL, get_string('checkcacheerrorsplit', 'tool_heartbeat', $splits)]; + } + + // Else OK. + return [result::OK, get_string('checkcachenotsplit', 'tool_heartbeat')]; + } + + /** + * Get the ping values from the cache and db to compare + * @param string $type type of check (e.g. web, cron) + */ + public function check($type) { + global $DB; + + $return = []; + $key = "checkcache{$type}ping"; + + // Read from cache (e.g. get_config uses cache). + $return[$type . 'api'] = get_config('tool_heartbeat', $key); + + // Read directly from database. + $return[$type . 'db'] = $DB->get_field('config_plugins', 'value', [ + 'plugin' => 'tool_heartbeat', + 'name' => $key, + ]); + return $return; + } + + /** + * Sets a timestamp in config from web or cron + * @param string $type type of check (e.g. web, cron) + */ + public static function ping($type) { + $key = "checkcache{$type}ping"; + $current = get_config('tool_heartbeat', $key); + + // Only update if the currently cached time is very old. + if ($current < (time() - DAYSECS) || true) { + debugging("HEARTBEAT doing {$type} ping {$current}", DEBUG_DEVELOPER); + set_config($key, time(), 'tool_heartbeat'); + } + } +} diff --git a/classes/task/cachecheck.php b/classes/task/cachecheck.php new file mode 100644 index 0000000..b5e57ff --- /dev/null +++ b/classes/task/cachecheck.php @@ -0,0 +1,46 @@ +. + + +namespace tool_heartbeat\task; + +/** + * Scheduled task to ping the cache from CRON. + * + * @package tool_heartbeat + * @author Brendan Heywood + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cachecheck extends \core\task\scheduled_task { + + /** + * Get task name + */ + public function get_name() { + return get_string('checkcachecheck', 'tool_heartbeat'); + } + + /** + * Execute task + */ + public function execute() { + \tool_heartbeat\check\cachecheck::ping('cron'); + } + +} + + diff --git a/db/install.php b/db/install.php new file mode 100644 index 0000000..6ec1c59 --- /dev/null +++ b/db/install.php @@ -0,0 +1,33 @@ +. +/** + * Cache split check. + * + * @package tool_heartbeat + * @author Brendan Heywood + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Install + */ +function xmldb_tool_heartbeat_install() { + // If there are issues with split caches they need to be exposed + // after some time for them to diverge. + \tool_heartbeat\check\cachecheck::ping('web'); + \tool_heartbeat\check\cachecheck::ping('cron'); +} diff --git a/db/tasks.php b/db/tasks.php new file mode 100644 index 0000000..f1bed20 --- /dev/null +++ b/db/tasks.php @@ -0,0 +1,36 @@ +. +/** + * Tool heartbeat + * + * @author Brendan Heywood + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$tasks = [ + [ + 'classname' => 'tool_heartbeat\task\cachecheck', + 'minute' => '*', + 'hour' => '*', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*', + ], +]; + diff --git a/db/upgrade.php b/db/upgrade.php new file mode 100644 index 0000000..c02b5e9 --- /dev/null +++ b/db/upgrade.php @@ -0,0 +1,38 @@ +. +/** + * DB upgrade script. + * + * @package tool_heartbeat + * @author Matthew Hilton + * @copyright Catalyst IT 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Upgrade + * @param int $oldversion + */ +function xmldb_tool_heartbeat_upgrade($oldversion) { + if ($oldversion < 2023102400) { + // If there are issues with split caches they need to be exposed + // after some time for them to diverge. + \tool_heartbeat\check\cachecheck::ping('web'); + \tool_heartbeat\check\cachecheck::ping('cron'); + } + + return true; +} diff --git a/lang/en/tool_heartbeat.php b/lang/en/tool_heartbeat.php index 4b2a029..6b8b22a 100644 --- a/lang/en/tool_heartbeat.php +++ b/lang/en/tool_heartbeat.php @@ -21,8 +21,6 @@ * @copyright 2014 Brendan Heywood * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ - -$string['pluginname'] = 'Heartbeat'; $string['allowedips'] = 'Allowed IPs Config'; $string['allowedipsdescription'] = 'Box to enter safe IP addresses for the heartbeat to respond to.'; $string['allowedipsempty'] = 'When the allowed IPs list is empty we will not block anyone. You can add your own IP address ({$a->ip}) and block all other IPs.'; @@ -34,6 +32,12 @@ $string['builtinallowediplist'] = 'Builtin IP Blocking Configuration'; $string['builtinallowediplist_desc'] = 'This allowed IP list would allow some IPs to be editable in the UI in addition to those forced in config.php.'; $string['checkauthcheck'] = 'Authentication methods'; +$string['checkcachecheck'] = 'Cache consistency check'; +$string['checkcachecronmissing'] = 'The cron cache check has not succeeded yet or is missing'; +$string['checkcachedetails'] = 'A split brain cache was detected. The value stored in the database table config_plugins was not the same as the cached value returned from get_config. If you purge the cache and this check passes and then fails again after a few hours then that strongly suggests a cache misconfiguration.'; +$string['checkcacheerrorsplit'] = 'The caches are not consistent: {$a}'; +$string['checkcachenotsplit'] = 'Caches appear consistent between web and cron'; +$string['checkcachewebmissing'] = 'The web cache check has not succeeded yet'; $string['checkdnscheck'] = 'DNS check'; $string['checklogstorebad'] = 'Logstore checks are bad! Please ensure at least one logstore has been set and enabled.'; $string['checklogstorecheck'] = 'Logstore check'; @@ -54,10 +58,12 @@ $string['errorlog'] = 'Error log period'; $string['errorlogdesc'] = 'To help ensure that all web server logging is working we can emit an intermittent message to the error_log. Set this to 0 to turn it off.'; $string['ips_combine'] = 'The IPs listed above will be combined with the IPs listed below.'; +$string["privacy:no_data_reason"] = "The Heartbeat plugin does not store any personal data."; $string['latencydelayedstart'] = 'Task {$a->task} start is delayed past configured threshold: {$a->mins}.'; $string['latencynotrun'] = 'Task {$a->task} has not run within the configured latency threshold: {$a->mins}.'; $string['latencyruntime'] = 'Task {$a->task} was last run with a runtime longer than the configured threshold: {$a->mins}.'; $string['normal'] = 'Normal monitoring'; +$string['pluginname'] = 'Heartbeat'; $string['progress'] = 'Progress bar test'; $string['progresshelp'] = 'This tests that all the various output buffers in the entire stack are corrent including but not limited to php, ob, gzip/deflat, varnish, nginx etc'; $string['setinitialauthstate'] = 'Initial auth state for heartbeat auth check set.'; @@ -69,7 +75,3 @@ $string['testing'] = 'Test heartbeat'; $string['testingdesc'] = 'You can use this to temporarily fake a warn or error condition to test that your monitoring is correctly working end to end.'; $string['testwarning'] = 'Fake a warning'; -/* - * Privacy provider (GDPR) - */ -$string["privacy:no_data_reason"] = "The Heartbeat plugin does not store any personal data."; diff --git a/lib.php b/lib.php index f248803..aa42e32 100644 --- a/lib.php +++ b/lib.php @@ -21,6 +21,13 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +/** + * Runs before HTTP headers. Used to ping the cachecheck. + */ +function tool_heartbeat_before_http_headers() { + \tool_heartbeat\check\cachecheck::ping('web'); +} + /** * Status checks. * @@ -29,6 +36,7 @@ function tool_heartbeat_status_checks() { return [ new \tool_heartbeat\check\authcheck(), + new \tool_heartbeat\check\cachecheck(), new \tool_heartbeat\check\logstorecheck(), new \tool_heartbeat\check\tasklatencycheck(), ]; diff --git a/version.php b/version.php index 2709306..9310ba5 100644 --- a/version.php +++ b/version.php @@ -24,8 +24,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023012700; -$plugin->release = 2023012700; // Match release exactly to version. +$plugin->version = 2023102400; +$plugin->release = 2202310240; // Match release exactly to version. $plugin->requires = 2012120311; // Deep support going back to 2.4. $plugin->supported = [24, 401]; $plugin->component = 'tool_heartbeat';