chore: update healthcheck termination

junkurihara · Dec 27, 2023 · da140e4 · da140e4
1 parent 4e59044
commit da140e4
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 2 deletions.
diff --git a/proxy-bin/src/main.rs b/proxy-bin/src/main.rs
@@ -106,7 +106,7 @@ async fn proxy_service_with_watcher(
   loop {
     tokio::select! {
       res = entrypoint(&proxy_conf, &runtime_handle, Some(term_notify.clone())) => {
-        error!("proxy entrypoint exited: {:?}", if res.is_err() { res.unwrap_err().to_string() } else { "".to_string() });
+        error!("proxy entrypoint exited: {}", if res.is_err() { res.unwrap_err().to_string() } else { "".to_string() });
         break;
       }
       _ = config_rx.changed() => {

diff --git a/proxy-lib/src/constants.rs b/proxy-lib/src/constants.rs
@@ -38,6 +38,12 @@ pub const ENDPOINT_RESOLUTION_PERIOD_MIN: u64 = 60;
 /// Health check: Check for health of paths and purge cache for every 600 secs
 pub const HEALTHCHECK_PERIOD_MIN: u64 = 10;
 
+/// Health check: Waiting time for health check retry when all possible paths are unhealthy
+pub const HEALTHCHECK_RETRY_WAITING_SEC: u64 = 10;
+
+/// Max retry for health check when all possible paths are unhealthy
+pub const MAX_ALL_UNHEALTHY_RETRY: usize = 5;
+
 /// Default DoH target server
 pub const DOH_TARGET_URL: &[&str] = &["https://dns.google/dns-query"];
 

diff --git a/proxy-lib/src/doh_client/doh_client_healthcheck.rs b/proxy-lib/src/doh_client/doh_client_healthcheck.rs
@@ -1,6 +1,8 @@
 use super::{dns_message, path_manage::DoHPath, DoHClient};
 use crate::{
-  constants::{HEALTHCHECK_TARGET_ADDR, HEALTHCHECK_TARGET_FQDN},
+  constants::{
+    HEALTHCHECK_RETRY_WAITING_SEC, HEALTHCHECK_TARGET_ADDR, HEALTHCHECK_TARGET_FQDN, MAX_ALL_UNHEALTHY_RETRY,
+  },
   error::*,
   log::*,
 };
@@ -36,6 +38,7 @@ impl DoHClient {
   /// - health of every path;
   /// - purge expired DNS cache
   async fn healthcheck_service(&self) -> Result<()> {
+    let mut all_unhealthy_cnt = 0;
     // purge expired DNS cache
     loop {
       info!("Execute periodic health check");
@@ -68,7 +71,13 @@ impl DoHClient {
         .flatten()
         .any(|v| v.is_healthy())
       {
+        all_unhealthy_cnt += 1;
         error!("All possible paths are unhealthy. Should check the Internet connection");
+        if all_unhealthy_cnt > MAX_ALL_UNHEALTHY_RETRY {
+          return Err(DapError::AllPathsUnhealthy);
+        }
+        tokio::time::sleep(tokio::time::Duration::from_secs(HEALTHCHECK_RETRY_WAITING_SEC)).await;
+        continue;
       }
       tokio::time::sleep(self.healthcheck_period_sec).await;
     }

diff --git a/proxy-lib/src/error.rs b/proxy-lib/src/error.rs
@@ -58,6 +58,9 @@ pub enum DapError {
   #[error("Invalid DNS response")]
   InvalidDnsResponse,
 
+  #[error("All paths are unhealthy even after some retry")]
+  AllPathsUnhealthy,
+
   #[error("No path available to send query")]
   NoPathAvailable,
   #[error("DoH query error")]