diff --git a/src/watcher/ftpparse.clj b/src/watcher/ftpparse.clj index 31d6262..d364261 100644 --- a/src/watcher/ftpparse.clj +++ b/src/watcher/ftpparse.clj @@ -187,5 +187,5 @@ (->> weekly-ftp-url fetch-ftp parse-ftp rest (filter (every-pred since? filename?)) vec))) (comment - (ftp-since #inst "2024-06-01")) + (ftp-since #inst "2024-07-01")) diff --git a/src/watcher/stream.clj b/src/watcher/stream.clj index 2400edd..23a0415 100644 --- a/src/watcher/stream.clj +++ b/src/watcher/stream.clj @@ -91,5 +91,6 @@ (comment (get-last-processed) - (save-to-topic (str (java.util.Date.)) (json/write-str [{"Name" "ClinVarRCVRelease_2024-0610.xml.gz", "Size" 4342574098, "Released" "2024-06-12 08:12:12", "Last Modified" "2024-06-12 08:12:12", "Directory" "/pub/clinvar/xml/RCV_release/weekly_release", "Host" "https://ftp.ncbi.nlm.nih.gov", "Release Date" "2024-06-12"}]))) + (save-to-topic (str (java.util.Date.)) (json/write-str [{"Name" "ClinVarVCVRelease_2024-0716.xml.gz", "Size" 3964266964, "Released" "2024-07-17 02:57:18", "Last Modified" "2024-07-17 02:57:18", "Directory" "/pub/clinvar/xml/weekly_release", "Host" "https://ftp.ncbi.nlm.nih.gov", "Release Date" "2024-07-16"}]))) + diff --git a/src/watcher/watcher.clj b/src/watcher/watcher.clj index a1ae25b..846eddf 100644 --- a/src/watcher/watcher.clj +++ b/src/watcher/watcher.clj @@ -74,7 +74,7 @@ [& args] (let [write-to-kafka (= -1 (if (some? args) (.indexOf args "--kafka") -1)) initiate-job (= -1 (if (some? args) (.indexOf args "--job") -1)) - files (-> (stream/get-last-processed) + files (-> (stream/get-last-processed) get-last-processed-date get-latest-files-since) file-details (process-file-details files) @@ -107,4 +107,10 @@ (comment - (-> (ftpparse/ftp-since #inst "2024-06-01"))) + (-> (ftpparse/ftp-since #inst "2024-07-01"))) + +(comment + (-> (stream/get-last-processed) + get-last-processed-date + get-latest-files-since + ))