From 625adf80d11c2ed0fb160acfaeca5976f4b0a70b Mon Sep 17 00:00:00 2001 From: d471061c Date: Mon, 9 Oct 2023 15:55:47 +0300 Subject: [PATCH 01/10] Change emails in sync --- services/headless-lms/models/src/users.rs | 18 ++++++++++++ .../server/src/programs/sync_tmc_users.rs | 29 ++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/services/headless-lms/models/src/users.rs b/services/headless-lms/models/src/users.rs index 7cc6fffc7f74..7d5bbb9b2d59 100644 --- a/services/headless-lms/models/src/users.rs +++ b/services/headless-lms/models/src/users.rs @@ -214,6 +214,24 @@ AND deleted_at IS NULL Ok(res.iter().map(|x| x.id).collect::>()) } +pub async fn update_email_for_user( + conn: &mut PgConnection, + id: Uuid, + new_email: String, +) -> ModelResult<()> { + info!("Updating user {id}"); + let mut tx = conn.begin().await?; + sqlx::query!( + "UPDATE users SET email_address = $1 WHERE id = $2", + new_email, + id, + ) + .execute(&mut *tx) + .await?; + info!("Email change succeeded"); + Ok(()) +} + pub async fn delete_user(conn: &mut PgConnection, id: Uuid) -> ModelResult<()> { info!("Deleting user {id}"); let mut tx = conn.begin().await?; diff --git a/services/headless-lms/server/src/programs/sync_tmc_users.rs b/services/headless-lms/server/src/programs/sync_tmc_users.rs index 780db90947a9..b4ed89085628 100644 --- a/services/headless-lms/server/src/programs/sync_tmc_users.rs +++ b/services/headless-lms/server/src/programs/sync_tmc_users.rs @@ -1,11 +1,12 @@ /*! Syncs tmc users */ -use std::env; +use std::{cmp::Ordering, env}; use crate::setup_tracing; use anyhow::Context; +use chrono::DateTime; use dotenv::dotenv; use headless_lms_models as models; use models::users::get_users_ids_in_db_from_upstream_ids; @@ -40,6 +41,32 @@ pub async fn main() -> anyhow::Result<()> { let db_pool = PgPool::connect(&database_url).await?; let mut conn = db_pool.acquire().await?; delete_users(&mut conn, &recent_changes).await?; + update_users(&mut conn, &recent_changes).await?; + Ok(()) +} + +pub async fn update_users( + conn: &mut PgConnection, + recent_changes: &TMCRecentChanges, +) -> anyhow::Result<()> { + let email_update_list = recent_changes + .changes + .iter() + .filter(|c| c.change_type == "email_changed") + .collect::>(); + + info!("Updating emails for {} users", email_update_list.len()); + email_update_list.sort_by(|a, b| { + DateTime::parse_from_rfc3339(a.created_at.as_str()) + .partial_cmp(DateTime::parse_from_rfc3339(b.created_at.as_str())) + .unwrap_or(Ordering::Equal) + }); + + for user in email_update_list { + update_email_for_user(&mut *conn, user.id, user.new_value); + } + + info!("Update done"); Ok(()) } From 5db0eeb8759f039fba078898a21886f29999f9b0 Mon Sep 17 00:00:00 2001 From: d471061c Date: Mon, 9 Oct 2023 16:01:12 +0300 Subject: [PATCH 02/10] Update sql-query for syncing --- services/headless-lms/models/src/users.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/headless-lms/models/src/users.rs b/services/headless-lms/models/src/users.rs index 7d5bbb9b2d59..74ddc0fc4907 100644 --- a/services/headless-lms/models/src/users.rs +++ b/services/headless-lms/models/src/users.rs @@ -222,7 +222,7 @@ pub async fn update_email_for_user( info!("Updating user {id}"); let mut tx = conn.begin().await?; sqlx::query!( - "UPDATE users SET email_address = $1 WHERE id = $2", + "UPDATE user_details SET email = $1 WHERE id = $2", new_email, id, ) From e0c33f03dd38c631ef51c6405068ba49fd34edb3 Mon Sep 17 00:00:00 2001 From: d471061c Date: Mon, 9 Oct 2023 16:02:07 +0300 Subject: [PATCH 03/10] Update sql-query for syncing --- services/headless-lms/models/src/users.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/headless-lms/models/src/users.rs b/services/headless-lms/models/src/users.rs index 74ddc0fc4907..24a56b59e6da 100644 --- a/services/headless-lms/models/src/users.rs +++ b/services/headless-lms/models/src/users.rs @@ -222,7 +222,7 @@ pub async fn update_email_for_user( info!("Updating user {id}"); let mut tx = conn.begin().await?; sqlx::query!( - "UPDATE user_details SET email = $1 WHERE id = $2", + "UPDATE user_details SET email = $1 WHERE user_id = $2", new_email, id, ) From 6b0788f75aac73a6cf32c4566157c260fb8eee81 Mon Sep 17 00:00:00 2001 From: d471061c Date: Tue, 10 Oct 2023 08:38:25 +0300 Subject: [PATCH 04/10] Use upstream_id instead --- services/headless-lms/models/src/users.rs | 24 ++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/services/headless-lms/models/src/users.rs b/services/headless-lms/models/src/users.rs index 24a56b59e6da..bd613b4363d7 100644 --- a/services/headless-lms/models/src/users.rs +++ b/services/headless-lms/models/src/users.rs @@ -216,18 +216,36 @@ AND deleted_at IS NULL pub async fn update_email_for_user( conn: &mut PgConnection, - id: Uuid, + upstream_id: Uuid, new_email: String, ) -> ModelResult<()> { - info!("Updating user {id}"); + info!("Updating user (Upstream id: {upstream_id})"); let mut tx = conn.begin().await?; + + let user = sqlx::query!( + "SELECT * FROM users_details WHERE upstream_id = $1", + upstream_id + ) + .fetch_one(&mut tx) + .await?; + sqlx::query!( "UPDATE user_details SET email = $1 WHERE user_id = $2", new_email, - id, + user.id, ) .execute(&mut *tx) .await?; + + let email_domain = new_email.split("@").nth(1); + sqlx::query!( + "UPDATE users SET email_domain = $1 WHERE id = $2", + email_domain, + user.id, + ) + .execute(&mut *tx) + .await?; + info!("Email change succeeded"); Ok(()) } From a078ca0afe2ef94390e52b6fd630cbbc56236a69 Mon Sep 17 00:00:00 2001 From: d471061c Date: Tue, 10 Oct 2023 09:03:36 +0300 Subject: [PATCH 05/10] verify user exists and update sqlx data --- services/headless-lms/models/sqlx-data.json | 3 +++ services/headless-lms/models/src/users.rs | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 services/headless-lms/models/sqlx-data.json diff --git a/services/headless-lms/models/sqlx-data.json b/services/headless-lms/models/sqlx-data.json new file mode 100644 index 000000000000..95c8c858baaf --- /dev/null +++ b/services/headless-lms/models/sqlx-data.json @@ -0,0 +1,3 @@ +{ + "db": "PostgreSQL" +} \ No newline at end of file diff --git a/services/headless-lms/models/src/users.rs b/services/headless-lms/models/src/users.rs index bd613b4363d7..c2dec48ee7b1 100644 --- a/services/headless-lms/models/src/users.rs +++ b/services/headless-lms/models/src/users.rs @@ -216,17 +216,18 @@ AND deleted_at IS NULL pub async fn update_email_for_user( conn: &mut PgConnection, - upstream_id: Uuid, + upstream_id: &i32, new_email: String, ) -> ModelResult<()> { info!("Updating user (Upstream id: {upstream_id})"); let mut tx = conn.begin().await?; - let user = sqlx::query!( - "SELECT * FROM users_details WHERE upstream_id = $1", + let user = sqlx::query_as!( + User, + "SELECT * FROM users WHERE upstream_id = $1", upstream_id ) - .fetch_one(&mut tx) + .fetch_one(&mut *tx) .await?; sqlx::query!( From 4d621e8d502fde6fbced9768ade1359edf3b68dc Mon Sep 17 00:00:00 2001 From: d471061c Date: Tue, 10 Oct 2023 11:14:06 +0300 Subject: [PATCH 06/10] Update sqlx-data --- ...04b5c3a5a30b9da3dfbf3127af33212ad058151ed2e1.json | 12 ++++++++++++ ...8684320ef265f050de1df5dcfa19c5ddf97ce24db72c.json | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 services/headless-lms/models/.sqlx/query-1ce8db09d26a1e35976f04b5c3a5a30b9da3dfbf3127af33212ad058151ed2e1.json create mode 100644 services/headless-lms/models/.sqlx/query-e6eb816ba83b975715cb8684320ef265f050de1df5dcfa19c5ddf97ce24db72c.json diff --git a/services/headless-lms/models/.sqlx/query-1ce8db09d26a1e35976f04b5c3a5a30b9da3dfbf3127af33212ad058151ed2e1.json b/services/headless-lms/models/.sqlx/query-1ce8db09d26a1e35976f04b5c3a5a30b9da3dfbf3127af33212ad058151ed2e1.json new file mode 100644 index 000000000000..f59c5053f0a0 --- /dev/null +++ b/services/headless-lms/models/.sqlx/query-1ce8db09d26a1e35976f04b5c3a5a30b9da3dfbf3127af33212ad058151ed2e1.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE user_details SET email = $1 WHERE user_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": ["Varchar", "Uuid"] + }, + "nullable": [] + }, + "hash": "1ce8db09d26a1e35976f04b5c3a5a30b9da3dfbf3127af33212ad058151ed2e1" +} diff --git a/services/headless-lms/models/.sqlx/query-e6eb816ba83b975715cb8684320ef265f050de1df5dcfa19c5ddf97ce24db72c.json b/services/headless-lms/models/.sqlx/query-e6eb816ba83b975715cb8684320ef265f050de1df5dcfa19c5ddf97ce24db72c.json new file mode 100644 index 000000000000..f31a5da72a43 --- /dev/null +++ b/services/headless-lms/models/.sqlx/query-e6eb816ba83b975715cb8684320ef265f050de1df5dcfa19c5ddf97ce24db72c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE users SET email_domain = $1 WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": ["Varchar", "Uuid"] + }, + "nullable": [] + }, + "hash": "e6eb816ba83b975715cb8684320ef265f050de1df5dcfa19c5ddf97ce24db72c" +} From 5dcfcf4518e629baeb66a34f86c01f1e26e5836c Mon Sep 17 00:00:00 2001 From: d471061c Date: Wed, 11 Oct 2023 18:26:20 +0300 Subject: [PATCH 07/10] fix ordering and style --- services/headless-lms/models/src/users.rs | 2 +- .../server/src/programs/sync_tmc_users.rs | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/services/headless-lms/models/src/users.rs b/services/headless-lms/models/src/users.rs index c2dec48ee7b1..0ee4f5f55cf8 100644 --- a/services/headless-lms/models/src/users.rs +++ b/services/headless-lms/models/src/users.rs @@ -238,7 +238,7 @@ pub async fn update_email_for_user( .execute(&mut *tx) .await?; - let email_domain = new_email.split("@").nth(1); + let email_domain = new_email.trim().split('@').last(); sqlx::query!( "UPDATE users SET email_domain = $1 WHERE id = $2", email_domain, diff --git a/services/headless-lms/server/src/programs/sync_tmc_users.rs b/services/headless-lms/server/src/programs/sync_tmc_users.rs index b4ed89085628..9b91c0db59fc 100644 --- a/services/headless-lms/server/src/programs/sync_tmc_users.rs +++ b/services/headless-lms/server/src/programs/sync_tmc_users.rs @@ -1,7 +1,7 @@ /*! Syncs tmc users */ -use std::{cmp::Ordering, env}; +use std::env; use crate::setup_tracing; use anyhow::Context; @@ -9,7 +9,7 @@ use anyhow::Context; use chrono::DateTime; use dotenv::dotenv; use headless_lms_models as models; -use models::users::get_users_ids_in_db_from_upstream_ids; +use models::users::{get_users_ids_in_db_from_upstream_ids, update_email_for_user}; use serde::{Deserialize, Serialize}; use sqlx::{PgConnection, PgPool}; @@ -49,7 +49,7 @@ pub async fn update_users( conn: &mut PgConnection, recent_changes: &TMCRecentChanges, ) -> anyhow::Result<()> { - let email_update_list = recent_changes + let mut email_update_list = recent_changes .changes .iter() .filter(|c| c.change_type == "email_changed") @@ -58,12 +58,17 @@ pub async fn update_users( info!("Updating emails for {} users", email_update_list.len()); email_update_list.sort_by(|a, b| { DateTime::parse_from_rfc3339(a.created_at.as_str()) - .partial_cmp(DateTime::parse_from_rfc3339(b.created_at.as_str())) - .unwrap_or(Ordering::Equal) + .unwrap() + .cmp(&DateTime::parse_from_rfc3339(b.created_at.as_str()).unwrap()) }); for user in email_update_list { - update_email_for_user(&mut *conn, user.id, user.new_value); + update_email_for_user( + &mut *conn, + &user.id, + user.new_value.as_deref().unwrap_or("unknown").to_string(), + ) + .await?; } info!("Update done"); From 0e502ecd680ae133f63bd13dafb5eb8c372e674d Mon Sep 17 00:00:00 2001 From: d471061c Date: Thu, 12 Oct 2023 13:51:44 +0300 Subject: [PATCH 08/10] add error handling and remove unused file --- services/headless-lms/models/sqlx-data.json | 3 -- .../server/src/programs/sync_tmc_users.rs | 33 ++++++++++++++++--- 2 files changed, 28 insertions(+), 8 deletions(-) delete mode 100644 services/headless-lms/models/sqlx-data.json diff --git a/services/headless-lms/models/sqlx-data.json b/services/headless-lms/models/sqlx-data.json deleted file mode 100644 index 95c8c858baaf..000000000000 --- a/services/headless-lms/models/sqlx-data.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "db": "PostgreSQL" -} \ No newline at end of file diff --git a/services/headless-lms/server/src/programs/sync_tmc_users.rs b/services/headless-lms/server/src/programs/sync_tmc_users.rs index 9b91c0db59fc..79caabe66889 100644 --- a/services/headless-lms/server/src/programs/sync_tmc_users.rs +++ b/services/headless-lms/server/src/programs/sync_tmc_users.rs @@ -57,18 +57,41 @@ pub async fn update_users( info!("Updating emails for {} users", email_update_list.len()); email_update_list.sort_by(|a, b| { - DateTime::parse_from_rfc3339(a.created_at.as_str()) - .unwrap() - .cmp(&DateTime::parse_from_rfc3339(b.created_at.as_str()).unwrap()) + let date_a = match DateTime::parse_from_rfc3339(a.created_at.as_str()) { + Ok(val) => val, + Err(e) => { + error!("Error converting date: '{}'", a.created_at); + error!("Error: {}", e); + DateTime::parse_from_str("01.01.1450", "%d.%m.%Y").unwrap() + } + }; + + let date_b = match DateTime::parse_from_rfc3339(b.created_at.as_str()) { + Ok(val) => val, + Err(e) => { + error!("Error converting date: '{}'", b.created_at); + error!("Error: {}", e); + DateTime::parse_from_str("01.01.1450", "%d.%m.%Y").unwrap() + } + }; + + date_a.cmp(&date_b) }); for user in email_update_list { - update_email_for_user( + match update_email_for_user( &mut *conn, &user.id, user.new_value.as_deref().unwrap_or("unknown").to_string(), ) - .await?; + .await + { + Ok(email) => email, + Err(e) => { + error!("Error updating user with id {}", user.id); + error!("Error: {}", e); + } + }; } info!("Update done"); From 36d3cc347ef68817c508ce8b6a4b060dac0b9e27 Mon Sep 17 00:00:00 2001 From: d471061c Date: Thu, 12 Oct 2023 15:07:35 +0300 Subject: [PATCH 09/10] add changes --- .../headless-lms/server/src/programs/sync_tmc_users.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/headless-lms/server/src/programs/sync_tmc_users.rs b/services/headless-lms/server/src/programs/sync_tmc_users.rs index 79caabe66889..5ca4fdf92042 100644 --- a/services/headless-lms/server/src/programs/sync_tmc_users.rs +++ b/services/headless-lms/server/src/programs/sync_tmc_users.rs @@ -78,17 +78,17 @@ pub async fn update_users( date_a.cmp(&date_b) }); - for user in email_update_list { + for change in email_update_list { match update_email_for_user( &mut *conn, - &user.id, - user.new_value.as_deref().unwrap_or("unknown").to_string(), + &change.user_id.unwrap(), + change.new_value.as_deref().unwrap_or("unknown").to_string(), ) .await { Ok(email) => email, Err(e) => { - error!("Error updating user with id {}", user.id); + error!("Error updating user with id {}", change.user_id.unwrap()); error!("Error: {}", e); } }; From 0729206b340c5c6bedfc3eef3f9eeed022661ad5 Mon Sep 17 00:00:00 2001 From: d471061c Date: Thu, 12 Oct 2023 15:22:08 +0300 Subject: [PATCH 10/10] remove unwraps --- .../server/src/programs/sync_tmc_users.rs | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/services/headless-lms/server/src/programs/sync_tmc_users.rs b/services/headless-lms/server/src/programs/sync_tmc_users.rs index 5ca4fdf92042..77e4fbcb204b 100644 --- a/services/headless-lms/server/src/programs/sync_tmc_users.rs +++ b/services/headless-lms/server/src/programs/sync_tmc_users.rs @@ -79,18 +79,20 @@ pub async fn update_users( }); for change in email_update_list { - match update_email_for_user( - &mut *conn, - &change.user_id.unwrap(), - change.new_value.as_deref().unwrap_or("unknown").to_string(), - ) - .await - { - Ok(email) => email, - Err(e) => { - error!("Error updating user with id {}", change.user_id.unwrap()); - error!("Error: {}", e); - } + if let Some(user_id) = change.user_id { + match update_email_for_user( + &mut *conn, + &user_id, + change.new_value.as_deref().unwrap_or("unknown").to_string(), + ) + .await + { + Ok(email) => email, + Err(e) => { + error!("Error updating user with id {}", user_id); + error!("Error: {}", e); + } + }; }; }