From 79d6be169be52024bb1b400de747653b6af2c60f Mon Sep 17 00:00:00 2001 From: amtoaer Date: Tue, 23 Jul 2024 01:44:59 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=90=84=E7=A7=8D?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=BB=93=E6=9D=9F=E4=B9=8B=E5=90=8E=E7=9A=84?= =?UTF-8?q?=20delay=20=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/bili_sync/src/adapter/collection.rs | 6 ++-- crates/bili_sync/src/adapter/favorite.rs | 6 ++-- crates/bili_sync/src/adapter/watch_later.rs | 6 ++-- crates/bili_sync/src/bilibili/collection.rs | 2 -- crates/bili_sync/src/config.rs | 35 ++++++++++++++++++++- crates/bili_sync/src/main.rs | 3 +- crates/bili_sync/src/utils/mod.rs | 20 ++++++++++++ crates/bili_sync/src/workflow.rs | 4 +++ 8 files changed, 68 insertions(+), 14 deletions(-) diff --git a/crates/bili_sync/src/adapter/collection.rs b/crates/bili_sync/src/adapter/collection.rs index c047d2c..6c91e1d 100644 --- a/crates/bili_sync/src/adapter/collection.rs +++ b/crates/bili_sync/src/adapter/collection.rs @@ -13,10 +13,10 @@ use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; use crate::adapter::VideoListModel; use crate::bilibili::{BiliClient, BiliError, Collection, CollectionItem, CollectionType, Video, VideoInfo}; -use crate::config::TEMPLATE; -use crate::utils::id_time_key; +use crate::config::{CONFIG, TEMPLATE}; use crate::utils::model::create_video_pages; use crate::utils::status::Status; +use crate::utils::{delay, id_time_key}; pub async fn collection_from<'a>( collection_item: &'a CollectionItem, @@ -176,9 +176,9 @@ impl VideoListModel for collection::Model { video_active_model.valid = Set(false); video_active_model.save(connection).await?; } - continue; } }; + delay(CONFIG.delay.fetch_video_detail.as_ref()).await; } Ok(()) } diff --git a/crates/bili_sync/src/adapter/favorite.rs b/crates/bili_sync/src/adapter/favorite.rs index 7d9d484..f4dffe3 100644 --- a/crates/bili_sync/src/adapter/favorite.rs +++ b/crates/bili_sync/src/adapter/favorite.rs @@ -13,10 +13,10 @@ use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; use crate::adapter::VideoListModel; use crate::bilibili::{BiliClient, BiliError, FavoriteList, Video, VideoInfo}; -use crate::config::TEMPLATE; -use crate::utils::id_time_key; +use crate::config::{CONFIG, TEMPLATE}; use crate::utils::model::create_video_pages; use crate::utils::status::Status; +use crate::utils::{delay, id_time_key}; pub async fn favorite_from<'a>( fid: &str, @@ -163,9 +163,9 @@ impl VideoListModel for favorite::Model { video_active_model.valid = Set(false); video_active_model.save(connection).await?; } - continue; } }; + delay(CONFIG.delay.fetch_video_detail.as_ref()).await; } Ok(()) } diff --git a/crates/bili_sync/src/adapter/watch_later.rs b/crates/bili_sync/src/adapter/watch_later.rs index 43e0d1c..15c90d2 100644 --- a/crates/bili_sync/src/adapter/watch_later.rs +++ b/crates/bili_sync/src/adapter/watch_later.rs @@ -13,10 +13,10 @@ use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; use crate::adapter::VideoListModel; use crate::bilibili::{BiliClient, BiliError, Video, VideoInfo, WatchLater}; -use crate::config::TEMPLATE; -use crate::utils::id_time_key; +use crate::config::{CONFIG, TEMPLATE}; use crate::utils::model::create_video_pages; use crate::utils::status::Status; +use crate::utils::{delay, id_time_key}; pub async fn watch_later_from<'a>( path: &Path, @@ -159,9 +159,9 @@ impl VideoListModel for watch_later::Model { video_active_model.valid = Set(false); video_active_model.save(connection).await?; } - continue; } }; + delay(CONFIG.delay.fetch_video_detail.as_ref()).await; } Ok(()) } diff --git a/crates/bili_sync/src/bilibili/collection.rs b/crates/bili_sync/src/bilibili/collection.rs index 8991616..0e62a91 100644 --- a/crates/bili_sync/src/bilibili/collection.rs +++ b/crates/bili_sync/src/bilibili/collection.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use std::fmt::{Display, Formatter}; use anyhow::Result; diff --git a/crates/bili_sync/src/config.rs b/crates/bili_sync/src/config.rs index 6b7bc45..a893304 100644 --- a/crates/bili_sync/src/config.rs +++ b/crates/bili_sync/src/config.rs @@ -76,6 +76,8 @@ pub struct Config { pub upper_path: PathBuf, #[serde(default)] pub nfo_time_type: NFOTimeType, + #[serde(default)] + pub delay: DelayConfig, } #[derive(Serialize, Deserialize, Default)] @@ -84,6 +86,21 @@ pub struct WatchLaterConfig { pub path: PathBuf, } +#[derive(Serialize, Deserialize, Default)] +pub struct DelayConfig { + pub refresh_video_list: Option, + pub fetch_video_detail: Option, + pub download_video: Option, + pub download_page: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(untagged, rename_all = "lowercase")] +pub enum Delay { + Random { min: u64, max: u64 }, + Fixed(u64), +} + #[derive(Serialize, Deserialize, Default)] #[serde(rename_all = "lowercase")] pub enum NFOTimeType { @@ -106,6 +123,7 @@ impl Default for Config { interval: 1200, upper_path: CONFIG_DIR.join("upper_face"), nfo_time_type: NFOTimeType::FavTime, + delay: Default::default(), } } } @@ -160,7 +178,22 @@ impl Config { error!("未设置 Credential 信息"); } } - + for delay_config in [ + &self.delay.refresh_video_list, + &self.delay.fetch_video_detail, + &self.delay.download_video, + &self.delay.download_page, + ] + .iter() + .filter_map(|x| x.as_ref()) + { + if let Delay::Random { min, max } = delay_config { + if min >= max { + ok = false; + error!("随机延迟的最小值应小于最大值"); + } + } + } if !ok { panic!( "位于 {} 的配置文件不合法，请参考提示信息修复后继续运行", diff --git a/crates/bili_sync/src/main.rs b/crates/bili_sync/src/main.rs index 3d1f618..551620f 100644 --- a/crates/bili_sync/src/main.rs +++ b/crates/bili_sync/src/main.rs @@ -9,7 +9,6 @@ mod downloader; mod error; mod utils; mod workflow; -use std::time::Duration; use once_cell::sync::Lazy; use tokio::time; @@ -74,6 +73,6 @@ async fn main() { info!("稍后再看处理完毕"); info!("本轮任务执行完毕，等待下一轮执行"); } - time::sleep(Duration::from_secs(CONFIG.interval)).await; + time::sleep(time::Duration::from_secs(CONFIG.interval)).await; } } diff --git a/crates/bili_sync/src/utils/mod.rs b/crates/bili_sync/src/utils/mod.rs index 7f8c230..81b5500 100644 --- a/crates/bili_sync/src/utils/mod.rs +++ b/crates/bili_sync/src/utils/mod.rs @@ -4,8 +4,12 @@ pub mod nfo; pub mod status; use chrono::{DateTime, Utc}; +use rand::Rng; +use tokio::time; use tracing_subscriber::util::SubscriberInitExt; +use crate::config::Delay; + pub fn init_logger(log_level: &str) { tracing_subscriber::fmt::Subscriber::builder() .with_env_filter(tracing_subscriber::EnvFilter::builder().parse_lossy(log_level)) @@ -21,3 +25,19 @@ pub fn init_logger(log_level: &str) { pub fn id_time_key(bvid: &String, time: &DateTime) -> String { format!("{}-{}", bvid, time.timestamp()) } + +pub(crate) async fn delay(delay: Option<&Delay>) { + match delay { + None => {} + Some(Delay::Random { min, max }) => { + let delay = { + let mut rng = rand::thread_rng(); + rng.gen_range(*min..=*max) + }; + time::sleep(time::Duration::from_millis(delay)).await; + } + Some(Delay::Fixed(delay)) => { + time::sleep(time::Duration::from_millis(*delay)).await; + } + } +} diff --git a/crates/bili_sync/src/workflow.rs b/crates/bili_sync/src/workflow.rs index 0195cf0..81755d0 100644 --- a/crates/bili_sync/src/workflow.rs +++ b/crates/bili_sync/src/workflow.rs @@ -18,6 +18,7 @@ use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, PageInfo, Vi use crate::config::{ARGS, CONFIG, TEMPLATE}; use crate::downloader::Downloader; use crate::error::{DownloadAbortError, ProcessPageError}; +use crate::utils::delay; use crate::utils::model::{create_videos, update_pages_model, update_videos_model}; use crate::utils::nfo::{ModelWrapper, NFOMode, NFOSerializer}; use crate::utils::status::{PageStatus, VideoStatus}; @@ -59,6 +60,7 @@ pub async fn refresh_video_list<'a>( info!("到达上一次处理的位置，提前中止"); break; } + delay(CONFIG.delay.refresh_video_list.as_ref()).await; } new_count = video_list_model.video_count(connection).await? - new_count; video_list_model.log_refresh_video_end(got_count, new_count); @@ -226,6 +228,7 @@ pub async fn download_video_pages( } let mut video_active_model: video::ActiveModel = video_model.into(); video_active_model.download_status = Set(status.into()); + delay(CONFIG.delay.download_video.as_ref()).await; Ok(video_active_model) } @@ -411,6 +414,7 @@ pub async fn download_page( let mut page_active_model: page::ActiveModel = page_model.into(); page_active_model.download_status = Set(status.into()); page_active_model.path = Set(Some(video_path.to_str().unwrap().to_string())); + delay(CONFIG.delay.download_page.as_ref()).await; Ok(page_active_model) }