[#135] Add cache consistency check

classes/check/cachecheck.php

@@ -0,0 +1,134 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_heartbeat\check;
+use core\check\check;
+use core\check\result;
+
+/**
+ * Cache check class
+ *
+ * This detects some split brain cache setups
+ *
+ * @package    tool_heartbeat
+ * @author     Brendan Heywood <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class cachecheck extends check {
+
+    /**
+     * Get Result.
+     *
+     * @return result
+     */
+    public function get_result() : result {
+        $results = $this->check('web');
+        $results += $this->check('cron');
+
+        list($status, $summary) = $this->build_result($results);
+
+        $details = '';
+
+        if ($status != result::OK) {
+            $details .= get_string('checkcachedetails', 'tool_heartbeat');
+        }
+
+        $details .= '<table class="table table-sm w-auto table-bordered">';
+
+        foreach ($results as $key => $value) {
+            $details .= \html_writer::start_tag('tr');
+            $details .= \html_writer::tag('td', $key);
+            $details .= \html_writer::tag('td', $value);
+
+            // Use DATE_RSS to show seconds, as well as timezone.
+            $details .= \html_writer::tag('td', date(DATE_RSS, $value));
+            $details .= \html_writer::end_tag('tr');
+        }
+        $details .= '</table>';
+        return new result($status, $summary, $details);
+    }
+
+    /**
+     * Reads the results and buils a check API result.
+     * @param array $results from check() function.
+     * @return array of [result status, summary string]
+     */
+    private function build_result(array $results): array {
+        // Nothing set for web API.
+        if (empty($results['webapi'])) {
+            return [result::CRITICAL, get_string('checkcachewebmissing', 'tool_heartbeat')];
+        }
+
+        // Nothing set for cron API.
+        if (empty($results['cronapi'])) {
+            return [result::CRITICAL, get_string('checkcachecronmissing', 'tool_heartbeat')];
+        }
+
+        // Check for split cron cache/db, web cache/db, and all of them together.
+        $cronsplit = $results['cronapi'] != $results['crondb'];
+        $websplit = $results['webapi'] != $results['webdb'];
+
+        if ($cronsplit || $websplit) {
+            $splits = [
+                'cron' => $cronsplit,
+                'web' => $websplit,
+            ];
+            $splits = implode(",", array_keys(array_filter($splits)));
+
+            return [result::CRITICAL, get_string('checkcacheerrorsplit', 'tool_heartbeat', $splits)];
+        }
+
+        // Else OK.
+        return [result::OK, get_string('checkcachenotsplit', 'tool_heartbeat')];
+    }
+
+    /**
+     * Get the ping values from the cache and db to compare
+     * @param string $type type of check (e.g. web, cron)
+     */
+    public function check($type) {
+        global $DB;
+
+        $return = [];
+        $key = "checkcache{$type}ping";
+
+        // Read from cache (e.g. get_config uses cache).
+        $return[$type . 'api'] = get_config('tool_heartbeat', $key);
+
+        // Read directly from database.
+        $return[$type . 'db'] = $DB->get_field('config_plugins', 'value', [
+            'plugin' => 'tool_heartbeat',
+            'name'  => $key,
+        ]);
+        return $return;
+    }
+
+    /**
+     * Sets a timestamp in config from web or cron
+     * @param string $type type of check (e.g. web, cron)
+     */
+    public static function ping($type) {
+        $key = "checkcache{$type}ping";
+        $current = get_config('tool_heartbeat', $key);
+
+        // Only update if the currently cached time is very old.
+        if ($current < (time() - DAYSECS)) {
+            debugging("HEARTBEAT doing {$type} ping {$current}", DEBUG_DEVELOPER);
+            set_config($key, time(), 'tool_heartbeat');
+        }
+    }
+}

classes/task/cachecheck.php

@@ -0,0 +1,48 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+
+namespace tool_heartbeat\task;
+
+/**
+ * Scheduled task to ping the cache from CRON.
+ *
+ * @package   tool_heartbeat
+ * @author    Brendan Heywood <[email protected]>
+ * @copyright Catalyst IT
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class cachecheck extends \core\task\scheduled_task {
+
+    /**
+     * Get task name
+     */
+    public function get_name() {
+        return get_string('checkcachecheck', 'tool_heartbeat');
+    }
+
+    /**
+     * Execute task
+     */
+    public function execute() {
+        if (class_exists('\core\check\manager')) {
+            \tool_heartbeat\check\cachecheck::ping('cron');
+        }
+    }
+
+}
+
+

db/install.php

@@ -0,0 +1,35 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+/**
+ * Cache split check.
+ *
+ * @package    tool_heartbeat
+ * @author     Brendan Heywood <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+/**
+ * Install
+ */
+function xmldb_tool_heartbeat_install() {
+    // If there are issues with split caches they need to be exposed
+    // after some time for them to diverge.
+    if (class_exists('\core\check\manager')) {
+        \tool_heartbeat\check\cachecheck::ping('web');
+        \tool_heartbeat\check\cachecheck::ping('cron');
+    }
+}

db/tasks.php

@@ -0,0 +1,36 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+/**
+ * Tool heartbeat
+ *
+ * @author     Brendan Heywood <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+$tasks = [
+    [
+        'classname' => 'tool_heartbeat\task\cachecheck',
+        'minute' => '*',
+        'hour' => '*',
+        'day' => '*',
+        'dayofweek' => '*',
+        'month' => '*',
+    ],
+];
+

db/upgrade.php

@@ -0,0 +1,40 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+/**
+ * DB upgrade script.
+ *
+ * @package    tool_heartbeat
+ * @author     Matthew Hilton <[email protected]>
+ * @copyright  Catalyst IT 2023
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+/**
+ * Upgrade
+ * @param int $oldversion
+ */
+function xmldb_tool_heartbeat_upgrade($oldversion) {
+    if ($oldversion < 2023102400) {
+        // If there are issues with split caches they need to be exposed
+        // after some time for them to diverge.
+        if (class_exists('\core\check\manager')) {
+            \tool_heartbeat\check\cachecheck::ping('web');
+            \tool_heartbeat\check\cachecheck::ping('cron');
+        }
+    }
+
+    return true;
+}

lang/en/tool_heartbeat.php

@@ -58,6 +58,12 @@
 $string['configuredauths'] = 'Check auth methods';
 $string['configuredauthsdesc'] = 'Auth methods to check are enabled in the Check API. A warning will be emitted if they are not enabled.';
 $string['checkauthcheck'] = 'Authentication methods';
+$string['checkcachecheck'] = 'Cache consistency check';
+$string['checkcachecronmissing'] = 'The cron cache check has not succeeded yet or is missing';
+$string['checkcachedetails'] = 'A split brain cache was detected. The value stored in the database table config_plugins was not the same as the cached value returned from get_config. If you purge the cache and this check passes and then fails again after a few hours then that strongly suggests a cache misconfiguration.';
+$string['checkcacheerrorsplit'] = 'The caches are not consistent: {$a}';
+$string['checkcachenotsplit'] = 'Caches appear consistent between web and cron';
+$string['checkcachewebmissing'] = 'The web cache check has not succeeded yet';
 $string['checkdnscheck'] = 'DNS check';
 $string['checkrangerequestcheck'] = 'Range requests check';
 $string['checkrangerequestok'] = 'Range requests are working, 206 response with only 10 bytes of data';

lib.php

@@ -21,6 +21,15 @@
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
+/**
+ * Runs before HTTP headers. Used to ping the cachecheck.
+ */
+function tool_heartbeat_before_http_headers() {
+    if (class_exists('\core\check\manager')) {
+        \tool_heartbeat\check\cachecheck::ping('web');
+    }
+}
+
 /**
  * Status checks.
  *
@@ -29,6 +38,7 @@
 function tool_heartbeat_status_checks() {
     return [
         new \tool_heartbeat\check\authcheck(),
+        new \tool_heartbeat\check\cachecheck(),
         new \tool_heartbeat\check\logstorecheck(),
         new \tool_heartbeat\check\tasklatencycheck(),
     ];