added progress chunk to limit logging (#2024)

* added progress chunk to limit logging

* lint

* comment based on suggestion

claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py

@@ -19,11 +19,13 @@ def missing_host_key(self, client, hostname, key):
         return
 
 
-def print_callback(filename, logger, bytes_so_far, bytes_total):
+def print_callback(filename, logger, bytes_so_far, bytes_total, progress_chunks):
     """Print the callback information."""
     rough_percent_transferred = int(100 * (bytes_so_far / bytes_total))
-    if (rough_percent_transferred % 25) == 0:
+    if rough_percent_transferred in progress_chunks:
         logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred)
+        # Remove progress chunk, so it is not logged again
+        progress_chunks.remove(rough_percent_transferred)
 
 OLD_FILENAME_TIMESTAMP = re.compile(
     r".*EDI_AGG_INPATIENT_[0-9]_(?P<ymd>[0-9]*)_(?P<hm>[0-9]*)[^0-9]*")
@@ -95,7 +97,8 @@ def download(ftp_credentials, out_path, logger):
 
     # download!
     for infile, outfile in filepaths_to_download.items():
-        callback_for_filename = functools.partial(print_callback, infile, logger)
+        callback_for_filename = functools.partial(print_callback, infile, logger, progress_chunks=[0, 25, 50, 75])
         sftp.get(infile, outfile, callback=callback_for_filename)
+        logger.info("Transfer finished", filename=infile, percent=100)
 
     client.close()

doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py

@@ -19,12 +19,13 @@ def missing_host_key(self, client, hostname, key):
         return
 
 
-def print_callback(filename, logger, bytes_so_far, bytes_total):
+def print_callback(filename, logger, bytes_so_far, bytes_total, progress_chunks):
     """Print the callback information."""
     rough_percent_transferred = int(100 * (bytes_so_far / bytes_total))
-    if (rough_percent_transferred % 25) == 0:
+    if rough_percent_transferred in progress_chunks:
         logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred)
-
+        # Remove progress chunk, so it is not logged again
+        progress_chunks.remove(rough_percent_transferred)
 
 OLD_FILENAME_TIMESTAMP = re.compile(
     r".*EDI_AGG_OUTPATIENT_[0-9]_(?P<ymd>[0-9]*)_(?P<hm>[0-9]*)[^0-9]*")
@@ -100,7 +101,8 @@ def download(ftp_credentials, out_path, logger, issue_date=None):
 
     # download!
     for infile, outfile in filepaths_to_download.items():
-        callback_for_filename = functools.partial(print_callback, infile, logger)
+        callback_for_filename = functools.partial(print_callback, infile, logger, progress_chunks=[0, 25, 50, 75])
         sftp.get(infile, outfile, callback=callback_for_filename)
+        logger.info("Transfer finished", filename=infile, percent=100)
 
     client.close()