Merge pull request #19 from clingen-data-model/somatic-topic-seed

Update misc for seeding watcher topic
clingen-data-model · Jul 23, 2024 · 379d263 · 379d263
2 parents 63f8592 + f1f7c2d
commit 379d263
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 4 deletions.
diff --git a/src/watcher/ftpparse.clj b/src/watcher/ftpparse.clj
@@ -187,5 +187,5 @@
     (->> weekly-ftp-url fetch-ftp parse-ftp rest (filter (every-pred since? filename?)) vec)))
 
 (comment
-  (ftp-since #inst "2024-06-01"))
+  (ftp-since #inst "2024-07-01"))
 
diff --git a/src/watcher/stream.clj b/src/watcher/stream.clj
@@ -91,5 +91,6 @@
 
 (comment
   (get-last-processed)
-  (save-to-topic (str (java.util.Date.)) (json/write-str [{"Name" "ClinVarRCVRelease_2024-0610.xml.gz", "Size" 4342574098, "Released" "2024-06-12 08:12:12", "Last Modified" "2024-06-12 08:12:12", "Directory" "/pub/clinvar/xml/RCV_release/weekly_release", "Host" "https://ftp.ncbi.nlm.nih.gov", "Release Date" "2024-06-12"}])))
+  (save-to-topic (str (java.util.Date.)) (json/write-str [{"Name" "ClinVarVCVRelease_2024-0716.xml.gz", "Size" 3964266964, "Released" "2024-07-17 02:57:18", "Last Modified" "2024-07-17 02:57:18", "Directory" "/pub/clinvar/xml/weekly_release", "Host" "https://ftp.ncbi.nlm.nih.gov", "Release Date" "2024-07-16"}])))
+
 
diff --git a/src/watcher/watcher.clj b/src/watcher/watcher.clj
@@ -74,7 +74,7 @@
   [& args]
   (let [write-to-kafka (= -1 (if (some? args) (.indexOf args "--kafka") -1))
         initiate-job (= -1 (if (some? args) (.indexOf args "--job") -1))
-        files (-> (stream/get-last-processed)
+        files  (-> (stream/get-last-processed)
                   get-last-processed-date
                   get-latest-files-since)
         file-details (process-file-details files)
@@ -107,4 +107,10 @@
 
 
 (comment
-  (-> (ftpparse/ftp-since #inst "2024-06-01")))
+  (-> (ftpparse/ftp-since #inst "2024-07-01")))
+
+(comment
+  (-> (stream/get-last-processed)
+                  get-last-processed-date
+                  get-latest-files-since
+                  ))