fix: Add pagination to list APIs (#209)

* fix: Add pagination to list APIs * fix: add fetch all support in list apis --------- Co-authored-by: Ankit Mahato <[email protected]>
juspay · Nov 13, 2024 · c155bb0 · c155bb0
1 parent 3f59418
commit c155bb0
Show file tree

Hide file tree

Showing 24 changed files with 606 additions and 262 deletions.
diff --git a/crates/context_aware_config/src/api/config/handlers.rs b/crates/context_aware_config/src/api/config/handlers.rs
@@ -34,7 +34,9 @@ use service_utils::{
 use superposition_macros::response_error;
 use superposition_macros::{bad_argument, db_error, unexpected_error};
 use superposition_types::{
-    custom_query::{self as superposition_query, CustomQuery, QueryFilters, QueryMap},
+    custom_query::{
+        self as superposition_query, CustomQuery, PaginationParams, QueryMap,
+    },
     result as superposition, Cac, Condition, Config, Context, Overrides,
     PaginatedResponse, TenantConfig, User,
 };
@@ -751,24 +753,33 @@ async fn get_resolved_config(
 #[get("/versions")]
 async fn get_config_versions(
     db_conn: DbConnection,
-    filters: Query<QueryFilters>,
+    filters: Query<PaginationParams>,
 ) -> superposition::Result<Json<PaginatedResponse<ConfigVersion>>> {
     let DbConnection(mut conn) = db_conn;
 
+    if let Some(true) = filters.all {
+        let config_versions: Vec<ConfigVersion> =
+            config_versions::config_versions.get_results(&mut conn)?;
+        return Ok(Json(PaginatedResponse {
+            total_pages: 1,
+            total_items: config_versions.len() as i64,
+            data: config_versions,
+        }));
+    }
+
     let n_version: i64 = config_versions::config_versions
         .count()
         .get_result(&mut conn)?;
+
+    let limit = filters.count.unwrap_or(10);
     let mut builder = config_versions::config_versions
         .into_boxed()
-        .order(config_versions::created_at.desc());
-    if let Some(limit) = filters.count {
-        builder = builder.limit(limit);
-    }
+        .order(config_versions::created_at.desc())
+        .limit(limit);
     if let Some(page) = filters.page {
-        let offset = (page - 1) * filters.count.unwrap_or(10);
+        let offset = (page - 1) * limit;
         builder = builder.offset(offset);
     }
-    let limit = filters.count.unwrap_or(10);
     let config_versions: Vec<ConfigVersion> = builder.load(&mut conn)?;
     let total_pages = (n_version as f64 / limit as f64).ceil() as i64;
     Ok(Json(PaginatedResponse {

diff --git a/crates/context_aware_config/src/api/default_config/handlers.rs b/crates/context_aware_config/src/api/default_config/handlers.rs
@@ -9,7 +9,9 @@ use service_utils::{
 use superposition_macros::{
     bad_argument, db_error, not_found, unexpected_error, validation_error,
 };
-use superposition_types::{result as superposition, User};
+use superposition_types::{
+    custom_query::PaginationParams, result as superposition, PaginatedResponse, User,
+};
 
 use crate::{
     api::{
@@ -30,7 +32,7 @@ use crate::helpers::put_config_in_redis;
 
 use actix_web::{
     delete, get, put,
-    web::{self, Data, Json, Path},
+    web::{self, Data, Json, Path, Query},
     HttpResponse, Scope,
 };
 use chrono::Utc;
@@ -211,11 +213,38 @@ fn fetch_default_key(
 }
 
 #[get("")]
-async fn get(db_conn: DbConnection) -> superposition::Result<Json<Vec<DefaultConfig>>> {
+async fn get(
+    db_conn: DbConnection,
+    filters: Query<PaginationParams>,
+) -> superposition::Result<Json<PaginatedResponse<DefaultConfig>>> {
     let DbConnection(mut conn) = db_conn;
 
-    let result: Vec<DefaultConfig> = dsl::default_configs.get_results(&mut conn)?;
-    Ok(Json(result))
+    if let Some(true) = filters.all {
+        let result: Vec<DefaultConfig> = dsl::default_configs.get_results(&mut conn)?;
+        return Ok(Json(PaginatedResponse {
+            total_pages: 1,
+            total_items: result.len() as i64,
+            data: result,
+        }));
+    }
+
+    let n_default_configs: i64 = dsl::default_configs.count().get_result(&mut conn)?;
+    let limit = filters.count.unwrap_or(10);
+    let mut builder = dsl::default_configs
+        .into_boxed()
+        .order(dsl::created_at.desc())
+        .limit(limit);
+    if let Some(page) = filters.page {
+        let offset = (page - 1) * limit;
+        builder = builder.offset(offset);
+    }
+    let result: Vec<DefaultConfig> = builder.load(&mut conn)?;
+    let total_pages = (n_default_configs as f64 / limit as f64).ceil() as i64;
+    Ok(Json(PaginatedResponse {
+        total_pages,
+        total_items: n_default_configs,
+        data: result,
+    }))
 }
 
 pub fn get_key_usage_context_ids(

diff --git a/crates/context_aware_config/src/api/dimension/handlers.rs b/crates/context_aware_config/src/api/dimension/handlers.rs
@@ -1,6 +1,14 @@
+use crate::{
+    api::dimension::{types::CreateReq, utils::get_dimension_usage_context_ids},
+    db::{
+        models::Dimension,
+        schema::{dimensions, dimensions::dsl::*},
+    },
+    helpers::validate_jsonschema,
+};
 use actix_web::{
     delete, get, put,
-    web::{self, Data, Path},
+    web::{self, Data, Json, Path, Query},
     HttpResponse, Scope,
 };
 use chrono::Utc;
@@ -11,17 +19,13 @@ use jsonschema::{Draft, JSONSchema};
 use serde_json::Value;
 use service_utils::service::types::{AppState, DbConnection};
 use superposition_macros::{bad_argument, not_found, unexpected_error};
-use superposition_types::{result as superposition, TenantConfig, User};
-
-use crate::{
-    api::dimension::{types::CreateReq, utils::get_dimension_usage_context_ids},
-    db::{
-        models::Dimension,
-        schema::{dimensions, dimensions::dsl::*},
-    },
-    helpers::validate_jsonschema,
+use superposition_types::{
+    custom_query::PaginationParams, result as superposition, PaginatedResponse,
+    TenantConfig, User,
 };
 
+extern crate base64;
+
 use super::types::{DeleteReq, DimensionWithMandatory};
 
 pub fn endpoints() -> Scope {
@@ -81,7 +85,7 @@ async fn create(
 
     let upsert = diesel::insert_into(dimensions)
         .values(&new_dimension)
-        .on_conflict(dimension)
+        .on_conflict(dimensions::dimension)
         .do_update()
         .set(&new_dimension)
         .get_result::<Dimension>(&mut conn);
@@ -119,10 +123,31 @@ async fn create(
 async fn get(
     db_conn: DbConnection,
     tenant_config: TenantConfig,
-) -> superposition::Result<HttpResponse> {
+    filters: Query<PaginationParams>,
+) -> superposition::Result<Json<PaginatedResponse<DimensionWithMandatory>>> {
     let DbConnection(mut conn) = db_conn;
 
-    let result: Vec<Dimension> = dimensions.get_results(&mut conn)?;
+    let (total_pages, total_items, result) = match filters.all {
+        Some(true) => {
+            let result: Vec<Dimension> = dimensions.get_results(&mut conn)?;
+            (1, result.len() as i64, result)
+        }
+        _ => {
+            let n_dimensions: i64 = dimensions.count().get_result(&mut conn)?;
+            let limit = filters.count.unwrap_or(10);
+            let mut builder = dimensions
+                .into_boxed()
+                .order(created_at.desc())
+                .limit(limit);
+            if let Some(page) = filters.page {
+                let offset = (page - 1) * limit;
+                builder = builder.offset(offset);
+            }
+            let result: Vec<Dimension> = builder.load(&mut conn)?;
+            let total_pages = (n_dimensions as f64 / limit as f64).ceil() as i64;
+            (total_pages, n_dimensions, result)
+        }
+    };
 
     let dimensions_with_mandatory: Vec<DimensionWithMandatory> = result
         .into_iter()
@@ -133,7 +158,11 @@ async fn get(
         })
         .collect();
 
-    Ok(HttpResponse::Ok().json(dimensions_with_mandatory))
+    Ok(Json(PaginatedResponse {
+        total_pages,
+        total_items,
+        data: dimensions_with_mandatory,
+    }))
 }
 
 #[delete("/{name}")]

diff --git a/crates/context_aware_config/src/api/functions/handlers.rs b/crates/context_aware_config/src/api/functions/handlers.rs
@@ -8,13 +8,13 @@ use crate::{
     db::{
         self,
         models::Function,
-        schema::functions::{dsl, dsl::functions, function_name},
+        schema::functions::{dsl, dsl::functions, function_name, last_modified_at},
     },
     validation_functions,
 };
 use actix_web::{
     delete, get, patch, post, put,
-    web::{self, Json, Path},
+    web::{self, Json, Path, Query},
     HttpResponse, Result, Scope,
 };
 use chrono::Utc;
@@ -23,7 +23,9 @@ use serde_json::json;
 use service_utils::service::types::DbConnection;
 
 use superposition_macros::{bad_argument, not_found, unexpected_error};
-use superposition_types::{result as superposition, User};
+use superposition_types::{
+    custom_query::PaginationParams, result as superposition, PaginatedResponse, User,
+};
 
 use validation_functions::{compile_fn, execute_fn};
 
@@ -170,13 +172,40 @@ async fn get(
 #[get("")]
 async fn list_functions(
     db_conn: DbConnection,
-) -> superposition::Result<Json<Vec<Function>>> {
+    filters: Query<PaginationParams>,
+) -> superposition::Result<Json<PaginatedResponse<Function>>> {
     let DbConnection(mut conn) = db_conn;
-    let mut function_list = functions.get_results(&mut conn)?;
-    for function in function_list.iter_mut() {
+
+    let (total_pages, total_items, mut data) = match filters.all {
+        Some(true) => {
+            let result: Vec<Function> = functions.get_results(&mut conn)?;
+            (1, result.len() as i64, result)
+        }
+        _ => {
+            let n_functions: i64 = functions.count().get_result(&mut conn)?;
+            let limit = filters.count.unwrap_or(10);
+            let mut builder = functions
+                .into_boxed()
+                .order(last_modified_at.desc())
+                .limit(limit);
+            if let Some(page) = filters.page {
+                let offset = (page - 1) * limit;
+                builder = builder.offset(offset);
+            }
+            let result: Vec<Function> = builder.load(&mut conn)?;
+            let total_pages = (n_functions as f64 / limit as f64).ceil() as i64;
+            (total_pages, n_functions, result)
+        }
+    };
+
+    for function in data.iter_mut() {
         decode_function(function)?;
     }
-    Ok(Json(function_list))
+    Ok(Json(PaginatedResponse {
+        total_pages,
+        total_items,
+        data,
+    }))
 }
 
 #[delete("/{function_name}")]

diff --git a/crates/context_aware_config/src/api/type_templates/handlers.rs b/crates/context_aware_config/src/api/type_templates/handlers.rs
@@ -7,15 +7,15 @@ use chrono::Utc;
 use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
 
 use jsonschema::JSONSchema;
-use serde_json::{json, Value};
+use serde_json::Value;
 use service_utils::service::types::DbConnection;
 use superposition_macros::{bad_argument, db_error};
-use superposition_types::{result as superposition, User};
-
-use crate::api::type_templates::types::{
-    QueryFilters, TypeTemplateName, TypeTemplateRequest,
+use superposition_types::{
+    custom_query::PaginationParams, result as superposition, PaginatedResponse, User,
 };
 
+use crate::api::type_templates::types::{TypeTemplateName, TypeTemplateRequest};
+
 pub fn endpoints() -> Scope {
     Scope::new("")
         .service(list_types)
@@ -119,29 +119,37 @@ async fn delete_type(
 #[get("")]
 async fn list_types(
     db_conn: DbConnection,
-    filters: Query<QueryFilters>,
-) -> superposition::Result<HttpResponse> {
+    filters: Query<PaginationParams>,
+) -> superposition::Result<Json<PaginatedResponse<TypeTemplates>>> {
     let DbConnection(mut conn) = db_conn;
 
+    if let Some(true) = filters.all {
+        let result: Vec<TypeTemplates> =
+            type_templates::dsl::type_templates.get_results(&mut conn)?;
+        return Ok(Json(PaginatedResponse {
+            total_pages: 1,
+            total_items: result.len() as i64,
+            data: result,
+        }));
+    };
+
     let n_types: i64 = type_templates::dsl::type_templates
         .count()
         .get_result(&mut conn)?;
+    let limit = filters.count.unwrap_or(10);
     let mut builder = type_templates::dsl::type_templates
         .into_boxed()
-        .order(type_templates::dsl::created_at.desc());
-    if let Some(limit) = filters.count {
-        builder = builder.limit(limit);
-    }
+        .order(type_templates::dsl::created_at.desc())
+        .limit(limit);
     if let Some(page) = filters.page {
-        let offset = (page - 1) * filters.count.unwrap_or(10);
+        let offset = (page - 1) * limit;
         builder = builder.offset(offset);
     }
-    let limit = filters.count.unwrap_or(10);
     let custom_types: Vec<TypeTemplates> = builder.load(&mut conn)?;
-    let total_pages = (n_types as f64 / limit as f64).ceil() as u64;
-    Ok(HttpResponse::Ok().json(json!({
-    "total_pages": total_pages,
-    "total_items": n_types,
-    "data": custom_types
-    })))
+    let total_pages = (n_types as f64 / limit as f64).ceil() as i64;
+    Ok(Json(PaginatedResponse {
+        total_pages,
+        total_items: n_types,
+        data: custom_types,
+    }))
 }
diff --git a/crates/context_aware_config/src/api/type_templates/types.rs b/crates/context_aware_config/src/api/type_templates/types.rs
@@ -17,11 +17,6 @@ pub struct TypeTemplateResponse {
     pub last_modified: String,
     pub created_by: String,
 }
-#[derive(Debug, Clone, Deserialize)]
-pub struct QueryFilters {
-    pub count: Option<i64>,
-    pub page: Option<i64>,
-}
 
 #[derive(Debug, Deserialize, Serialize, AsRef, Deref, DerefMut, Into, Clone)]
 #[serde(try_from = "String")]