Add default header check when --head-before-get is not set

src/cli/sync.rs

@@ -19,7 +19,7 @@ use tracing::{debug, error, info, warn};
 use url::Url;
 
 use crate::{
-    compare::{should_download_by_head, should_download_by_list},
+    compare::{should_download_by_header, should_download_by_list},
     extensions::{extension_handler, ExtensionPackage},
     listing::{self, ListItem},
     parser::ListResult,
@@ -133,14 +133,16 @@ fn determinate_timezone(
 }
 
 fn download_file(
-    async_context: &AsyncContext,
+    task_context: &TaskContext,
     item: &ListItem,
     path: &Path,
     args: &SyncArgs,
     mprogress: &MultiProgress,
-    timezone: Option<FixedOffset>,
     cwd: &Path,
+    check_header: bool,
 ) -> Result<()> {
+    let async_context = task_context.async_context;
+    let timezone = task_context.timezone;
     let client = &async_context.download_client;
     let runtime = &async_context.runtime;
     // Here we use async to allow streaming and progress bar
@@ -155,6 +157,10 @@ fn download_file(
                     return Err(e);
                 }
             };
+            if check_header && !should_download_by_header(path, &resp, false) {
+                warn!("Skipping {} (GET header matches local file)", url);
+                return Ok(());
+            }
             let total_size = match resp.content_length() {
                 Some(s) => s,
                 None => {
@@ -432,7 +438,7 @@ fn download_handler(
             args.retry,
         ) {
             Ok(resp) => {
-                if !should_download_by_head(&expected_path, &resp, compare_size_only) {
+                if !should_download_by_header(&expected_path, &resp, compare_size_only) {
                     info!("Skipping (by HEAD) {}", task.url);
                     should_download = false;
                 }
@@ -449,13 +455,14 @@ fn download_handler(
 
     if should_download && !args.dry_run {
         if (download_file(
-            task_context.async_context,
+            task_context,
             item,
             &expected_path,
             args,
             mprogress,
-            task_context.timezone,
             cwd,
+            // If no sending HEAD before GET, check header here
+            !args.head_before_get,
         ))
         .is_err()
         {

src/compare.rs

@@ -96,7 +96,7 @@ pub fn should_download_by_list(
     }
 }
 
-pub fn should_download_by_head(path: &Path, resp: &reqwest::Response, size_only: bool) -> bool {
+pub fn should_download_by_header(path: &Path, resp: &reqwest::Response, size_only: bool) -> bool {
     // Construct a valid "ListItem" and pass to should_download_by_list
     debug!("Checking {:?} by HEAD: {:?}", path, resp);
     let item = ListItem {
@@ -107,11 +107,28 @@ pub fn should_download_by_head(path: &Path, resp: &reqwest::Response, size_only:
         } else {
             FileType::File
         },
-        size: Some(FileSize::Precise(
-            resp.content_length()
-                .expect("No content-length from upstream"),
-        )),
-        mtime: utils::get_response_mtime(resp).unwrap().naive_utc(),
+        size: Some(FileSize::Precise(match resp.content_length() {
+            Some(l) => l,
+            None => {
+                warn!(
+                    "No content-length from upstream ({}), go downloading anyway",
+                    resp.url()
+                );
+                return true;
+            }
+        })),
+        mtime: match utils::get_response_mtime(resp) {
+            Ok(m) => m,
+            Err(e) => {
+                warn!(
+                    "Cannot get mtime from {} ({}), go downloading anyway",
+                    resp.url(),
+                    e
+                );
+                return true;
+            }
+        }
+        .naive_utc(),
         timezone: None,
         skip_check: false,
     };

src/main.rs

@@ -99,7 +99,8 @@ pub struct SyncArgs {
     #[clap(long, default_value_t = 3)]
     retry: usize,
 
-    /// Do an HEAD before actual GET. Add this if you are not sure if the results from parser is correct.
+    /// Do an HEAD before actual GET.
+    /// Otherwise, when GETting tsumugu would try checking if we still need to download it...
     #[clap(long)]
     head_before_get: bool,