[#135] Added split cache check

catalyst · Oct 23, 2023 · 2a01fee · 2a01fee
1 parent f741324
commit 2a01fee
Show file tree

Hide file tree

Showing 8 changed files with 305 additions and 8 deletions.
diff --git a/classes/check/cachecheck.php b/classes/check/cachecheck.php
@@ -0,0 +1,134 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_heartbeat\check;
+use core\check\check;
+use core\check\result;
+
+/**
+ * Cache check class
+ *
+ * This detects some split brain cache setups
+ *
+ * @package    tool_heartbeat
+ * @author     Brendan Heywood <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class cachecheck extends check {
+
+    /**
+     * Get Result.
+     *
+     * @return result
+     */
+    public function get_result() : result {
+        $results = $this->check('web');
+        $results += $this->check('cron');
+
+        list($status, $summary) = $this->build_result($results);
+
+        $details = '';
+
+        if ($status != result::OK) {
+            $details .= get_string('checkcachedetails', 'tool_heartbeat');
+        }
+
+        $details .= '<table class="table table-sm w-auto table-bordered">';
+
+        foreach ($results as $key => $value) {
+            $details .= \html_writer::start_tag('tr');
+            $details .= \html_writer::tag('td', $key);
+            $details .= \html_writer::tag('td', $value);
+
+            // Use DATE_RSS to show seconds, as well as timezone.
+            $details .= \html_writer::tag('td', date(DATE_RSS, $value));
+            $details .= \html_writer::end_tag('tr');
+        }
+        $details .= '</table>';
+        return new result($status, $summary, $details);
+    }
+
+    /**
+     * Reads the results and buils a check API result.
+     * @param array $results from check() function.
+     * @return array of [result status, summary string]
+     */
+    private function build_result(array $results): array {
+        // Nothing set for web API.
+        if (empty($results['webapi'])) {
+            return [result::CRITICAL, get_string('checkcachewebmissing', 'tool_heartbeat')];
+        }
+
+        // Nothing set for cron API.
+        if (empty($results['cronapi'])) {
+            return [result::CRITICAL, get_string('checkcachecronmissing', 'tool_heartbeat')];
+        }
+
+        // Check for split cron cache/db, web cache/db, and all of them together.
+        $cronsplit = $results['cronapi'] != $results['crondb'];
+        $websplit = $results['webapi'] != $results['webdb'];
+
+        if ($cronsplit || $websplit) {
+            $splits = [
+                'cron' => $cronsplit,
+                'web' => $websplit,
+            ];
+            $splits = implode(",", array_keys(array_filter($splits)));
+
+            return [result::CRITICAL, get_string('checkcacheerrorsplit', 'tool_heartbeat', $splits)];
+        }
+
+        // Else OK.
+        return [result::OK, get_string('checkcachenotsplit', 'tool_heartbeat')];
+    }
+
+    /**
+     * Get the ping values from the cache and db to compare
+     * @param string $type type of check (e.g. web, cron)
+     */
+    public function check($type) {
+        global $DB;
+
+        $return = [];
+        $key = "checkcache{$type}ping";
+
+        // Read from cache (e.g. get_config uses cache).
+        $return[$type . 'api'] = get_config('tool_heartbeat', $key);
+
+        // Read directly from database.
+        $return[$type . 'db'] = $DB->get_field('config_plugins', 'value', [
+            'plugin' => 'tool_heartbeat',
+            'name'  => $key,
+        ]);
+        return $return;
+    }
+
+    /**
+     * Sets a timestamp in config from web or cron
+     * @param string $type type of check (e.g. web, cron)
+     */
+    public static function ping($type) {
+        $key = "checkcache{$type}ping";
+        $current = get_config('tool_heartbeat', $key);
+
+        // Only update if the currently cached time is very old.
+        if ($current < (time() - DAYSECS) || true) {
+            debugging("HEARTBEAT doing {$type} ping {$current}", DEBUG_DEVELOPER);
+            set_config($key, time(), 'tool_heartbeat');
+        }
+    }
+}
diff --git a/classes/task/cachecheck.php b/classes/task/cachecheck.php
@@ -0,0 +1,46 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+
+namespace tool_heartbeat\task;
+
+/**
+ * Scheduled task to ping the cache from CRON.
+ *
+ * @package   tool_heartbeat
+ * @author    Brendan Heywood <[email protected]>
+ * @copyright Catalyst IT
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class cachecheck extends \core\task\scheduled_task {
+
+    /**
+     * Get task name
+     */
+    public function get_name() {
+        return get_string('checkcachecheck', 'tool_heartbeat');
+    }
+
+    /**
+     * Execute task
+     */
+    public function execute() {
+        \tool_heartbeat\check\cachecheck::ping('cron');
+    }
+
+}
+
+
diff --git a/db/install.php b/db/install.php
@@ -0,0 +1,33 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+/**
+ * Cache split check.
+ *
+ * @package    tool_heartbeat
+ * @author     Brendan Heywood <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+/**
+ * Install
+ */
+function xmldb_tool_heartbeat_install() {
+    // If there are issues with split caches they need to be exposed
+    // after some time for them to diverge.
+    \tool_heartbeat\check\cachecheck::ping('web');
+    \tool_heartbeat\check\cachecheck::ping('cron');
+}
diff --git a/db/tasks.php b/db/tasks.php
@@ -0,0 +1,36 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+/**
+ * Tool heartbeat
+ *
+ * @author     Brendan Heywood <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+$tasks = [
+    [
+        'classname' => 'tool_heartbeat\task\cachecheck',
+        'minute' => '*',
+        'hour' => '*',
+        'day' => '*',
+        'dayofweek' => '*',
+        'month' => '*',
+    ],
+];
+
diff --git a/db/upgrade.php b/db/upgrade.php
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+/**
+ * DB upgrade script.
+ *
+ * @package    tool_heartbeat
+ * @author     Matthew Hilton <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+/**
+ * Upgrade
+ * @param int $oldversion
+ */
+function xmldb_tool_heartbeat_upgrade($oldversion) {
+    if ($oldversion < 2023102400) {
+        // If there are issues with split caches they need to be exposed
+        // after some time for them to diverge.
+        \tool_heartbeat\check\cachecheck::ping('web');
+        \tool_heartbeat\check\cachecheck::ping('cron');
+    }
+
+    return true;
+}
diff --git a/lang/en/tool_heartbeat.php b/lang/en/tool_heartbeat.php
@@ -21,8 +21,6 @@
  * @copyright  2014 Brendan Heywood <[email protected]>
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
-
-$string['pluginname'] = 'Heartbeat';
 $string['allowedips'] = 'Allowed IPs Config';
 $string['allowedipsdescription'] = 'Box to enter safe IP addresses for the heartbeat to respond to.';
 $string['allowedipsempty'] = 'When the allowed IPs list is empty we will not block anyone. You can add your own IP address (<i>{$a->ip}</i>) and block all other IPs.';
@@ -34,6 +32,12 @@
 $string['builtinallowediplist'] = 'Builtin IP Blocking Configuration';
 $string['builtinallowediplist_desc'] = 'This allowed IP list would allow some IPs to be editable in the UI in addition to those forced in config.php.';
 $string['checkauthcheck'] = 'Authentication methods';
+$string['checkcachecheck'] = 'Cache consistency check';
+$string['checkcachecronmissing'] = 'The cron cache check has not succeeded yet or is missing';
+$string['checkcachedetails'] = 'A split brain cache was detected. The value stored in the database table config_plugins was not the same as the cached value returned from get_config. If you purge the cache and this check passes and then fails again after a few hours then that strongly suggests a cache misconfiguration.';
+$string['checkcacheerrorsplit'] = 'The caches are not consistent: {$a}';
+$string['checkcachenotsplit'] = 'Caches appear consistent between web and cron';
+$string['checkcachewebmissing'] = 'The web cache check has not succeeded yet';
 $string['checkdnscheck'] = 'DNS check';
 $string['checklogstorebad'] = 'Logstore checks are bad! Please ensure at least one logstore has been set and enabled.';
 $string['checklogstorecheck'] = 'Logstore check';
@@ -54,10 +58,12 @@
 $string['errorlog'] = 'Error log period';
 $string['errorlogdesc'] = 'To help ensure that all web server logging is working we can emit an intermittent message to the error_log. Set this to 0 to turn it off.';
 $string['ips_combine'] = 'The IPs listed above will be combined with the IPs listed below.';
+$string["privacy:no_data_reason"] = "The Heartbeat plugin does not store any personal data.";
 $string['latencydelayedstart'] = 'Task {$a->task} start is delayed past configured threshold: {$a->mins}.';
 $string['latencynotrun'] = 'Task {$a->task} has not run within the configured latency threshold: {$a->mins}.';
 $string['latencyruntime'] = 'Task {$a->task} was last run with a runtime longer than the configured threshold: {$a->mins}.';
 $string['normal'] = 'Normal monitoring';
+$string['pluginname'] = 'Heartbeat';
 $string['progress'] = 'Progress bar test';
 $string['progresshelp'] = 'This tests that all the various output buffers in the entire stack are corrent including but not limited to php, ob, gzip/deflat, varnish, nginx etc';
 $string['setinitialauthstate'] = 'Initial auth state for heartbeat auth check set.';
@@ -69,7 +75,3 @@
 $string['testing'] = 'Test heartbeat';
 $string['testingdesc'] = 'You can use this to temporarily fake a warn or error condition to test that your monitoring is correctly working end to end.';
 $string['testwarning'] = 'Fake a warning';
-/*
- * Privacy provider (GDPR)
- */
-$string["privacy:no_data_reason"] = "The Heartbeat plugin does not store any personal data.";
diff --git a/lib.php b/lib.php
@@ -21,6 +21,13 @@
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
+/**
+ * Runs before HTTP headers. Used to ping the cachecheck.
+ */
+function tool_heartbeat_before_http_headers() {
+    \tool_heartbeat\check\cachecheck::ping('web');
+}
+
 /**
  * Status checks.
  *
@@ -29,6 +36,7 @@
 function tool_heartbeat_status_checks() {
     return [
         new \tool_heartbeat\check\authcheck(),
+        new \tool_heartbeat\check\cachecheck(),
         new \tool_heartbeat\check\logstorecheck(),
         new \tool_heartbeat\check\tasklatencycheck(),
     ];

diff --git a/version.php b/version.php
@@ -24,8 +24,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2023012700;
-$plugin->release   = 2023012700; // Match release exactly to version.
+$plugin->version   = 2023102400;
+$plugin->release   = 2202310240; // Match release exactly to version.
 $plugin->requires  = 2012120311; // Deep support going back to 2.4.
 $plugin->supported = [24, 401];
 $plugin->component = 'tool_heartbeat';