FabBackend: fail fast on unresponsive backend

fablab-bergamo · Aug 16, 2024 · 4a8ca30 · 4a8ca30
1 parent ef7535f
commit 4a8ca30
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 9 deletions.
diff --git a/conf/conf.hpp b/conf/conf.hpp
@@ -148,11 +148,14 @@ namespace fabomatic
     static constexpr std::string_view request_topic{"/request"};
 
     /// @brief Number of tries to get a reply from the backend
-    static constexpr auto MAX_TRIES{2};
+    static constexpr auto MAX_TRIES{1};
 
     /// @brief Timeout for a single backend reply request.
     static constexpr auto TIMEOUT_REPLY_SERVER{2s};
 
+    /// @brief Once backend is unresponsive, wait at least this period before to try again
+    static constexpr auto FAIL_FAST_PERIOD {45s};
+
     /// @brief MQTT port for broker
     static constexpr auto PORT_NUMBER{1883};
 

diff --git a/include/FabBackend.hpp b/include/FabBackend.hpp
@@ -89,6 +89,7 @@ namespace fabomatic
     [[nodiscard]] auto hasBufferedMsg() const -> bool;
     [[nodiscard]] auto transmitBuffer() -> bool;
     [[nodiscard]] auto saveBuffer() -> bool;
+    [[nodiscard]] auto shouldFailFast() const -> bool;
 
     [[nodiscard]] auto checkBackendRequest() -> std::optional<std::unique_ptr<MQTTInterface::BackendRequest>>;
 

diff --git a/src/AuthProvider.cpp b/src/AuthProvider.cpp
@@ -60,7 +60,12 @@ namespace fabomatic
     ESP_LOGD(TAG, "tryLogin called for %s", uid_str.c_str());
 
     if (!server.isOnline())
-      server.connect();
+    {
+      if (!server.shouldFailFast())
+      {
+        server.connect();
+      }
+    }
 
     if (server.isOnline())
     {

diff --git a/src/FabBackend.cpp b/src/FabBackend.cpp
@@ -457,12 +457,17 @@ namespace fabomatic
       // To preserve chronological order, we cannot send new messages until the old ones have been sent.
       ESP_LOGW(TAG, "Online with pending messages that could not be transmitted, retrying...");
 
-      Tasks::delay(250ms);
-
-      if (!isOnline())
+      if (!shouldFailFast())
       {
         connect();
       }
+      else
+      {
+        ESP_LOGW(TAG, "processQuery: failing fast due to unresponsive backend.");
+        break;
+      }
+
+      Tasks::delay(250ms);
       nb_tries++;
     }
 
@@ -514,12 +519,20 @@ namespace fabomatic
       // To preserve chronological order, we cannot send new messages until the old ones have been sent.
       ESP_LOGW(TAG, "Online with pending messages that could not be transmitted, retrying...");
 
-      Tasks::delay(250ms);
-
       if (!isOnline())
       {
-        connect();
+        if (!shouldFailFast())
+        {
+          connect();
+        }
+        else
+        {
+          ESP_LOGW(TAG, "processQuery: failing fast due to unresponsive backend.");
+          break;
+        }
       }
+
+      Tasks::delay(250ms);
       nb_tries++;
     }
 
@@ -721,4 +734,14 @@ namespace fabomatic
     return std::nullopt;
   }
 
+  /**
+   * @brief when server is unresponsive, wait for a configurable time before to try again
+   */
+  auto FabBackend::shouldFailFast() const -> bool
+  {
+    return this->mqtt_connected &&
+           this->last_unresponsive.has_value() &&
+           (Tasks::arduinoNow() - this->last_unresponsive.value()) <= conf::mqtt::FAIL_FAST_PERIOD;
+  }
+
 } // namespace fabomatic
diff --git a/test/test_mqtt/test_mqtt.cpp b/test/test_mqtt/test_mqtt.cpp
@@ -39,7 +39,7 @@ namespace fabomatic::tests
 
   void busyWait(std::chrono::seconds d = 3s)
   {
-    auto start = fabomatic::Tasks::arduinoNow();
+    const auto start = fabomatic::Tasks::arduinoNow();
     while (fabomatic::Tasks::arduinoNow() - start <= d)
     {
       test_scheduler.execute();
@@ -367,6 +367,8 @@ namespace fabomatic::tests
     // Renable backend
     broker.generateReplies(true);
 
+    TEST_ASSERT_TRUE_MESSAGE(server.connect(), "(degraded) force reconnection");
+
     auto alive_resp = server.alive();
     TEST_ASSERT_TRUE_MESSAGE(alive_resp, "(degraded) Server alive failed");
     TEST_ASSERT_FALSE_MESSAGE(server.hasBufferedMsg(), "(degraded) Buffered messages not sent");