Optional date fields and stronger types (#22)

This PR introduces two improvements:

    Optional Date fields in ExecutionTimes Struct (since some of these fields do not exist for all stages of query execution).
    Stronger types on our refresh testing. We now have proper float and date result parsing.

README.md

@@ -9,24 +9,23 @@ cargo add duners
 ```
 
 ```rust
-use std::env;
-use dotenv::dotenv;
-use duners::client::{DuneClient, DuneRequestError};
+use chrono::{DateTime, Utc};
+use duners::{client::DuneClient, dateutil::datetime_from_str};
 use serde::Deserialize;
 
 // User must declare the expected query return fields and types!
 #[derive(Deserialize, Debug, PartialEq)]
 struct ResultStruct {
     text_field: String,
-    number_field: String,
-    date_field: String,
+    number_field: f64,
+    #[serde(deserialize_with = "datetime_from_str")]
+    date_field: DateTime<Utc>,
     list_field: String,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), DuneRequestError> {
-    dotenv().ok();
-    let dune = DuneClient::new(env::var("DUNE_API_KEY").unwrap());
+    let dune = DuneClient::from_env();
     let results = dune.refresh::<ResultStruct>(1215383, None, None).await?;
     println!("{:?}", results.get_rows());
     Ok(())

src/client.rs

@@ -1,13 +1,15 @@
+use crate::error::{DuneError, DuneRequestError};
 use crate::parameters::Parameter;
 use crate::response::{
-    CancellationResponse, DuneError, ExecutionResponse, ExecutionStatus, GetResultResponse,
-    GetStatusResponse,
+    CancellationResponse, ExecutionResponse, ExecutionStatus, GetResultResponse, GetStatusResponse,
 };
+use dotenv::dotenv;
 use log::{debug, error, info, warn};
 use reqwest::{Error, Response};
 use serde::de::DeserializeOwned;
 use serde_json::json;
 use std::collections::HashMap;
+use std::env;
 use tokio::time::{sleep, Duration};
 
 const BASE_URL: &str = "https://api.dune.com/api/v1";
@@ -32,33 +34,18 @@ pub struct DuneClient {
     api_key: String,
 }
 
-#[derive(Debug, PartialEq)]
-pub enum DuneRequestError {
-    /// Includes known errors:
-    /// "invalid API Key"
-    /// "Query not found"
-    /// "The requested execution ID (ID: wonky job ID) is invalid."
-    Dune(String),
-    /// Errors bubbled up from reqwest::Error
-    Request(String),
-}
-
-impl From<DuneError> for DuneRequestError {
-    fn from(value: DuneError) -> Self {
-        DuneRequestError::Dune(value.error)
-    }
-}
-
-impl From<Error> for DuneRequestError {
-    fn from(value: Error) -> Self {
-        DuneRequestError::Request(value.to_string())
-    }
-}
-
 impl DuneClient {
     /// Constructor
-    pub fn new(api_key: String) -> Self {
-        DuneClient { api_key }
+    pub fn new(api_key: &str) -> DuneClient {
+        DuneClient {
+            api_key: api_key.to_string(),
+        }
+    }
+    pub fn from_env() -> DuneClient {
+        dotenv().ok();
+        DuneClient {
+            api_key: env::var("DUNE_API_KEY").unwrap(),
+        }
     }
 
     /// Internal POST request handler
@@ -171,24 +158,27 @@ impl DuneClient {
     ///
     /// # Examples
     /// ```
-    /// use std::env;
-    /// use dotenv::dotenv;
-    /// use duners::client::{DuneClient, DuneRequestError};
+    /// use duners::{
+    ///     client::DuneClient,
+    ///     dateutil::datetime_from_str,
+    ///     error::DuneRequestError
+    /// };
     /// use serde::Deserialize;
+    /// use chrono::{DateTime, Utc};
     ///
     /// // User must declare the expected query return types and fields.
     /// #[derive(Deserialize, Debug, PartialEq)]
     /// struct ResultStruct {
     ///     text_field: String,
-    ///     number_field: String,
-    ///     date_field: String,
+    ///     number_field: f64,
+    ///     #[serde(deserialize_with = "datetime_from_str")]
+    ///     date_field: DateTime<Utc>,
     ///     list_field: String,
     /// }
     ///
     /// #[tokio::main]
     /// async fn main() -> Result<(), DuneRequestError> {
-    ///     dotenv().ok();
-    ///     let dune = DuneClient::new(env::var("DUNE_API_KEY").unwrap());
+    ///     let dune = DuneClient::from_env();
     ///     let results = dune.refresh::<ResultStruct>(1215383, None, None).await?;
     ///     println!("{:?}", results.get_rows());
     ///     Ok(())
@@ -201,6 +191,7 @@ impl DuneClient {
         ping_frequency: Option<u64>,
     ) -> Result<GetResultResponse<T>, DuneRequestError> {
         let job_id = self.execute_query(query_id, parameters).await?.execution_id;
+        info!("Refreshing {} Execution ID {}", query_id, job_id);
         let mut status = self.get_status(&job_id).await?;
         while !status.state.is_terminal() {
             info!(
@@ -224,36 +215,17 @@ impl DuneClient {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::dateutil::{date_parse, datetime_from_str};
     use crate::response::ExecutionStatus;
-    use crate::util::date_parse;
-    use dotenv::dotenv;
+    use chrono::{DateTime, Utc};
     use serde::Deserialize;
-    use std::env;
 
     const QUERY_ID: u32 = 971694;
     const JOB_ID: &str = "01GMZ8R4NPPQZCWYJRY2K03MH0";
 
-    fn get_dune() -> DuneClient {
-        dotenv().ok();
-        DuneClient {
-            api_key: env::var("DUNE_API_KEY").unwrap(),
-        }
-    }
-
-    #[tokio::test]
-    async fn error_parsing() {
-        let err = reqwest::get("invalid-url").await.unwrap_err();
-        assert_eq!(
-            DuneRequestError::from(err),
-            DuneRequestError::Request("builder error: relative URL without a base".to_string())
-        );
-    }
-
     #[tokio::test]
     async fn invalid_api_key() {
-        let dune = DuneClient {
-            api_key: "Baloney".parse().unwrap(),
-        };
+        let dune = DuneClient::new("Baloney");
         let error = dune.execute_query(QUERY_ID, None).await.unwrap_err();
         assert_eq!(
             error,
@@ -263,7 +235,7 @@ mod tests {
 
     #[tokio::test]
     async fn invalid_query_id() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
         let error = dune.execute_query(u32::MAX, None).await.unwrap_err();
         assert_eq!(
             error,
@@ -273,7 +245,7 @@ mod tests {
 
     #[tokio::test]
     async fn invalid_job_id() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
         let error = dune
             .get_results::<DuneError>("wonky job ID")
             .await
@@ -288,7 +260,7 @@ mod tests {
 
     #[tokio::test]
     async fn execute_query() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
         let exec = dune.execute_query(QUERY_ID, None).await.unwrap();
         // Also testing cancellation!
         let cancellation = dune.cancel_execution(&exec.execution_id).await.unwrap();
@@ -297,7 +269,7 @@ mod tests {
 
     #[tokio::test]
     async fn execute_query_with_params() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
         let all_parameter_types = vec![
             Parameter::date("DateField", date_parse("2022-05-04T00:00:00.0Z").unwrap()),
             Parameter::number("NumberField", "3.1415926535"),
@@ -310,14 +282,14 @@ mod tests {
 
     #[tokio::test]
     async fn get_status() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
         let status = dune.get_status(JOB_ID).await.unwrap();
         assert_eq!(status.state, ExecutionStatus::Complete)
     }
 
     #[tokio::test]
     async fn get_results() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
 
         #[derive(Deserialize, Debug)]
         struct ExpectedResults {
@@ -337,27 +309,44 @@ mod tests {
 
     #[tokio::test]
     async fn refresh() {
-        let dune = get_dune();
+        let dune = DuneClient::from_env();
 
         #[derive(Deserialize, Debug, PartialEq)]
         struct ResultStruct {
             text_field: String,
-            number_field: String,
-            date_field: String,
+            number_field: f64,
+            #[serde(deserialize_with = "datetime_from_str")]
+            date_field: DateTime<Utc>,
             list_field: String,
         }
         let results = dune
-            .refresh::<ResultStruct>(1215383, None, None)
+            .refresh::<ResultStruct>(
+                1215383,
+                Some(vec![Parameter::number("NumberField", "3.141592653589793")]),
+                None,
+            )
             .await
             .unwrap();
         assert_eq!(
             ResultStruct {
                 text_field: "Plain Text".to_string(),
-                number_field: "3.1415926535".to_string(),
-                date_field: "2022-05-04 00:00:00".to_string(),
+                number_field: std::f64::consts::PI,
+                date_field: date_parse("2022-05-04T00:00:00.0Z").unwrap(),
                 list_field: "Option 1".to_string(),
             },
             results.get_rows()[0]
         )
     }
+
+    #[tokio::test]
+    #[ignore]
+    async fn long_running_query() {
+        let dune = DuneClient::from_env();
+        let results = dune
+            .refresh::<HashMap<String, f64>>(1229120, None, None)
+            .await
+            .unwrap();
+        println!("Job ID {:?}", results.execution_id);
+        assert_eq!(results.state, ExecutionStatus::Complete);
+    }
 }

src/dateutil.rs

@@ -0,0 +1,63 @@
+#![allow(dead_code)]
+use chrono::{DateTime, NaiveDateTime, ParseError, Utc};
+use serde::{de, Deserialize, Deserializer};
+
+fn date_string_parser(date_str: &str, format: &str) -> Result<DateTime<Utc>, ParseError> {
+    let native = NaiveDateTime::parse_from_str(date_str, format);
+    Ok(DateTime::<Utc>::from_utc(native?, Utc))
+}
+
+/// The date format returned by DuneAPI response Date fields (e.g. `submitted_at`)
+pub fn date_parse(date_str: &str) -> Result<DateTime<Utc>, ParseError> {
+    date_string_parser(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
+}
+
+/// The Date format returned from data fields of type timestamp.
+pub fn dune_date(date_str: &str) -> Result<DateTime<Utc>, ParseError> {
+    date_string_parser(date_str, "%Y-%m-%dT%H:%M:%S")
+}
+
+pub fn datetime_from_str<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s: String = Deserialize::deserialize(deserializer)?;
+    match date_parse(&s) {
+        // First try to parse response type date strings
+        Ok(parsed_date) => Ok(parsed_date),
+        Err(_) => {
+            // First attempt didn't work, try another format
+            dune_date(&s).map_err(de::Error::custom)
+        }
+    }
+}
+
+pub fn optional_datetime_from_str<'de, D>(
+    deserializer: D,
+) -> Result<Option<DateTime<Utc>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s: Option<String> = Deserialize::deserialize(deserializer)?;
+    match s {
+        None => Ok(None),
+        Some(s) => {
+            let date = date_parse(&s).map_err(de::Error::custom)?;
+            Ok(Some(date))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn date_parse_works() {
+        let date_str = "2022-01-01T01:02:03.123Z";
+        assert_eq!(
+            date_parse(date_str).unwrap().to_string(),
+            "2022-01-01 01:02:03.000000123 UTC"
+        )
+    }
+}