[change] Stop sending checksum requests on 404 #160

Implements and closes #160

Co-authored-by: Federico Capoano <[email protected]>

.github/workflows/ci.yml

@@ -85,6 +85,7 @@ jobs:
           path: "${{ env.DOWNLOADS_DIR }}/${{ env.START_TIME }}"
 
       - name: Setup Google Cloud
+        if: ${{ github.event_name=='push' }}
         uses: google-github-actions/setup-gcloud@master
         with:
           service_account_key: ${{ secrets.GCS_DOWNLOADS_SERVICE_ACCOUNT_JSON }}

README.rst

@@ -102,6 +102,12 @@ UCI configuration options must go in ``/etc/config/openwisp``.
 - ``respawn_threshold``: time in seconds used as procd respawn threshold, defaults to ``3600``
 - ``respawn_timeout``: time in seconds used as procd respawn timeout, defaults to ``5``
 - ``respawn_retry``: number of procd respawn retries (use ``0`` for infinity), defaults to ``5``
+- ``checksum_max_retries``: maximum number of retries for checksum requests which fail with 404, defaults to ``5``,
+  after these failures the agent will assume the device has been deleted from OpenWISP Controller and will exit;
+  please keep in mind that due to ``respawn_retry``, procd will try to respawn the agent after it exits, so the
+  total number of attempts which will be tried has to be calculated as:
+  ``checksum_max_retries * respawn_retry``
+- ``checksum_retry_delay``: time in seconds between retries, defaults to ``6``
 
 Automatic registration
 ----------------------

openwisp-config/files/openwisp.agent

@@ -121,6 +121,14 @@ while [ -n "$1" ]; do
 			export POST_REGISTRATION_HOOK="$2"
 			shift
 			;;
+		--checksum-max-retries)
+			export CHECKSUM_MAX_RETRIES="$2"
+			shift
+			;;
+		--checksum-retry-delay)
+			export CHECKSUM_RETRY_DELAY="$2"
+			shift
+			;;
 		-*)
 			echo "Invalid option: $1"
 			exit 1
@@ -186,6 +194,8 @@ BOOTUP="$WORKING_DIR/bootup"
 REGISTRATION_URL="$URL/controller/register/"
 UNMANAGED_DIR="$WORKING_DIR/unmanaged"
 FETCH_COMMAND="curl -s --connect-timeout $CONNECT_TIMEOUT --max-time $MAX_TIME"
+CHECKSUM_MAX_RETRIES=${CHECKSUM_MAX_RETRIES:-5}
+CHECKSUM_RETRY_DELAY=${CHECKSUM_RETRY_DELAY:-6}
 mkdir -p $WORKING_DIR
 mkdir -p $UNMANAGED_DIR
 
@@ -358,26 +368,52 @@ register() {
 }
 
 # gets checksum from controller
+# - 404 trigger retries and if the 404 persists the agent
+#   quits assuming the device is deleted from OpenWISP Controller
+# - any other error is logged but the agent continues execution
+#   and will eventually try again at the next cycle
 get_checksum() {
-	$FETCH_COMMAND -i "$CHECKSUM_URL" >"$1"
 	local exit_code status
-	exit_code=$?
 
-	if [ "$exit_code" -ne "0" ]; then
-		logger -s "Failed to connect to controller while getting checksum: curl exit code $exit_code" \
-			-t openwisp \
-			-p daemon.err
-		return 2
-	fi
+	for attempt_no in $(seq 1 "$CHECKSUM_MAX_RETRIES"); do
+		$FETCH_COMMAND -i "$CHECKSUM_URL" >"$1"
+		exit_code=$?
+
+		if [ "$exit_code" -ne "0" ]; then
+			logger -s "Failed to connect to controller while getting checksum: curl exit code $exit_code" \
+				-t openwisp \
+				-p daemon.err
+			return 2
+		fi
+
+		if is_http_status "$1" 404; then
+			logger -s "Failed to retrieve checksum: 404 Not Found" \
+				-t openwisp \
+				-p daemon.warning
+
+			if [ "$attempt_no" -eq "$CHECKSUM_MAX_RETRIES" ]; then
+				logger -s "Giving up and shutting down: the device may have been deleted from OpenWISP Controller" \
+					-t openwisp \
+					-p daemon.err
+				exit 0
+			fi
+
+			sleep "$CHECKSUM_RETRY_DELAY"
+			# try again
+			continue
+		else
+			break
+		fi
+	done
 
 	if ! is_http_status "$1" 200; then
-		local status
 		status=$(head -n 1 "$1")
 		logger -s "Failed to retrieve checksum: $status" \
 			-t openwisp \
 			-p daemon.err
 		return 3
 	fi
+
 	check_header "$1"
 }
 

openwisp-config/files/openwisp.config

@@ -25,6 +25,8 @@ config controller 'http'
 	#option max_time '30'
 	#option capath '/etc/ssl/certs'
 	#option cacert '/etc/ssl/certs/ca-certificates.crt'
+	#option checksum_max_retries '5'
+	#option checksum_retry_delay '6'
 	# hooks
 	#option pre_reload_hook '/usr/sbin/my_pre_reload_hook'
 	#option post_reload_hook '/usr/sbin/my_post_reload_hook'

openwisp-config/files/openwisp.init

@@ -50,6 +50,8 @@ parse_config() {
 	add_option "$cfg" "--post-reload-hook" post_reload_hook
 	add_option "$cfg" "--post-reload-delay" post_reload_delay
 	add_option "$cfg" "--post-registration-hook" post_registration_hook
+	add_option "$cfg" "--checksum-max-retries" checksum_max_retries
+	add_option "$cfg" "--checksum-retry-delay" checksum_retry_delay
 }
 
 start_service() {