diff --git a/crates/bili_sync/src/config.rs b/crates/bili_sync/src/config.rs deleted file mode 100644 index 4606015..0000000 --- a/crates/bili_sync/src/config.rs +++ /dev/null @@ -1,308 +0,0 @@ -use std::borrow::Cow; -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::Arc; - -use anyhow::Result; -use arc_swap::ArcSwapOption; -use handlebars::handlebars_helper; -use once_cell::sync::Lazy; -use serde::de::{Deserializer, MapAccess, Visitor}; -use serde::ser::SerializeMap; -use serde::{Deserialize, Serialize}; - -use crate::bilibili::{CollectionItem, CollectionType, Credential, DanmakuOption, FilterOption}; - -pub static TEMPLATE: Lazy = Lazy::new(|| { - let mut handlebars = handlebars::Handlebars::new(); - handlebars_helper!(truncate: |s: String, len: usize| { - if s.chars().count() > len { - s.chars().take(len).collect::() - } else { - s.to_string() - } - }); - handlebars.register_helper("truncate", Box::new(truncate)); - handlebars - .register_template_string("video", &CONFIG.video_name) - .unwrap(); - handlebars.register_template_string("page", &CONFIG.page_name).unwrap(); - handlebars -}); - -pub static CONFIG: Lazy = Lazy::new(|| { - let config = Config::load().unwrap_or_else(|err| { - if err - .downcast_ref::() - .map_or(true, |e| e.kind() != std::io::ErrorKind::NotFound) - { - panic!("加载配置文件失败,错误为: {err}"); - } - warn!("配置文件不存在,使用默认配置..."); - Config::default() - }); - // 放到外面,确保新的配置项被保存 - info!("配置加载完毕,覆盖刷新原有配置"); - config.save().unwrap(); - // 检查配置文件内容 - info!("校验配置文件内容..."); - config.check(); - config -}); - -pub static ARGS: Lazy = Lazy::new(Args::parse); - -pub static CONFIG_DIR: Lazy = - Lazy::new(|| dirs::config_dir().expect("No config path found").join("bili-sync")); - -#[derive(Serialize, Deserialize)] -pub struct Config { - pub credential: ArcSwapOption, - pub filter_option: FilterOption, - #[serde(default)] - pub danmaku_option: DanmakuOption, - pub favorite_list: HashMap, - #[serde( - default, - serialize_with = "serialize_collection_list", - deserialize_with = "deserialize_collection_list" - )] - pub collection_list: HashMap, - #[serde(default)] - pub submission_list: HashMap, - #[serde(default)] - pub watch_later: WatchLaterConfig, - pub video_name: Cow<'static, str>, - pub page_name: Cow<'static, str>, - pub interval: u64, - pub upper_path: PathBuf, - #[serde(default)] - pub nfo_time_type: NFOTimeType, - #[serde(default)] - pub delay: DelayConfig, - #[serde(default = "default_time_format")] - pub time_format: String, -} - -fn default_time_format() -> String { - "%Y-%m-%d".to_string() -} - -#[derive(Serialize, Deserialize, Default)] -pub struct WatchLaterConfig { - pub enabled: bool, - pub path: PathBuf, -} - -#[derive(Serialize, Deserialize, Default)] -pub struct DelayConfig { - pub refresh_video_list: Option, - pub fetch_video_detail: Option, - pub download_video: Option, - pub download_page: Option, -} - -#[derive(Serialize, Deserialize)] -#[serde(untagged, rename_all = "lowercase")] -pub enum Delay { - Random { min: u64, max: u64 }, - Fixed(u64), -} - -#[derive(Serialize, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum NFOTimeType { - #[default] - FavTime, - PubTime, -} - -impl Default for Config { - fn default() -> Self { - Self { - credential: ArcSwapOption::from(Some(Arc::new(Credential::default()))), - filter_option: FilterOption::default(), - danmaku_option: DanmakuOption::default(), - favorite_list: HashMap::new(), - collection_list: HashMap::new(), - submission_list: HashMap::new(), - watch_later: Default::default(), - video_name: Cow::Borrowed("{{title}}"), - page_name: Cow::Borrowed("{{bvid}}"), - interval: 1200, - upper_path: CONFIG_DIR.join("upper_face"), - nfo_time_type: NFOTimeType::FavTime, - delay: Default::default(), - time_format: default_time_format(), - } - } -} - -impl Config { - /// 简单的预检查 - pub fn check(&self) { - let mut ok = true; - if self.favorite_list.is_empty() && self.collection_list.is_empty() && !self.watch_later.enabled { - ok = false; - error!("没有配置任何需要扫描的内容,程序空转没有意义"); - } - if self.watch_later.enabled && !self.watch_later.path.is_absolute() { - error!( - "稍后再看保存的路径应为绝对路径,检测到:{}", - self.watch_later.path.display() - ); - } - for path in self.favorite_list.values() { - if !path.is_absolute() { - ok = false; - error!("收藏夹保存的路径应为绝对路径,检测到: {}", path.display()); - } - } - if !self.upper_path.is_absolute() { - ok = false; - error!("up 主头像保存的路径应为绝对路径"); - } - if self.video_name.is_empty() { - ok = false; - error!("未设置 video_name 模板"); - } - if self.page_name.is_empty() { - ok = false; - error!("未设置 page_name 模板"); - } - let credential = self.credential.load(); - match credential.as_deref() { - Some(credential) => { - if credential.sessdata.is_empty() - || credential.bili_jct.is_empty() - || credential.buvid3.is_empty() - || credential.dedeuserid.is_empty() - || credential.ac_time_value.is_empty() - { - ok = false; - error!("Credential 信息不完整,请确保填写完整"); - } - } - None => { - ok = false; - error!("未设置 Credential 信息"); - } - } - for delay_config in [ - &self.delay.refresh_video_list, - &self.delay.fetch_video_detail, - &self.delay.download_video, - &self.delay.download_page, - ] - .iter() - .filter_map(|x| x.as_ref()) - { - if let Delay::Random { min, max } = delay_config { - if min >= max { - ok = false; - error!("随机延迟的最小值应小于最大值"); - } - } - } - if !ok { - panic!( - "位于 {} 的配置文件不合法,请参考提示信息修复后继续运行", - CONFIG_DIR.join("config.toml").display() - ); - } - } - - fn load() -> Result { - let config_path = CONFIG_DIR.join("config.toml"); - let config_content = std::fs::read_to_string(config_path)?; - Ok(toml::from_str(&config_content)?) - } - - pub fn save(&self) -> Result<()> { - let config_path = CONFIG_DIR.join("config.toml"); - std::fs::create_dir_all(&*CONFIG_DIR)?; - std::fs::write(config_path, toml::to_string_pretty(self)?)?; - Ok(()) - } -} - -fn serialize_collection_list( - collection_list: &HashMap, - serializer: S, -) -> Result -where - S: serde::Serializer, -{ - let mut map = serializer.serialize_map(Some(collection_list.len()))?; - for (k, v) in collection_list { - let prefix = match k.collection_type { - CollectionType::Series => "series", - CollectionType::Season => "season", - }; - map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?; - } - map.end() -} - -fn deserialize_collection_list<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct CollectionListVisitor; - - impl<'de> Visitor<'de> for CollectionListVisitor { - type Value = HashMap; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a map of collection list") - } - - fn visit_map(self, mut map: A) -> Result - where - A: MapAccess<'de>, - { - let mut collection_list = HashMap::new(); - while let Some((key, value)) = map.next_entry::()? { - let collection_item = match key.split(':').collect::>().as_slice() { - [prefix, mid, sid] => { - let collection_type = match *prefix { - "series" => CollectionType::Series, - "season" => CollectionType::Season, - _ => { - return Err(serde::de::Error::custom( - "invalid collection type, should be series or season", - )) - } - }; - CollectionItem { - mid: mid.to_string(), - sid: sid.to_string(), - collection_type, - } - } - _ => { - return Err(serde::de::Error::custom( - "invalid collection key, should be series:mid:sid or season:mid:sid", - )) - } - }; - collection_list.insert(collection_item, value); - } - Ok(collection_list) - } - } - - deserializer.deserialize_map(CollectionListVisitor) -} - -use clap::Parser; - -#[derive(Parser)] -#[command(version, about, long_about = None)] -pub struct Args { - #[arg(short, long, env = "SCAN_ONLY")] - pub scan_only: bool, - - #[arg(short, long, default_value = "None,bili_sync=info", env = "RUST_LOG")] - pub log_level: String, -} diff --git a/crates/bili_sync/src/config/clap.rs b/crates/bili_sync/src/config/clap.rs new file mode 100644 index 0000000..342e50f --- /dev/null +++ b/crates/bili_sync/src/config/clap.rs @@ -0,0 +1,11 @@ +use clap::Parser; + +#[derive(Parser)] +#[command(version, about, long_about = None)] +pub struct Args { + #[arg(short, long, env = "SCAN_ONLY")] + pub scan_only: bool, + + #[arg(short, long, default_value = "None,bili_sync=info", env = "RUST_LOG")] + pub log_level: String, +} diff --git a/crates/bili_sync/src/config/global.rs b/crates/bili_sync/src/config/global.rs new file mode 100644 index 0000000..4f0eccb --- /dev/null +++ b/crates/bili_sync/src/config/global.rs @@ -0,0 +1,54 @@ +use std::path::PathBuf; + +use clap::Parser; +use handlebars::handlebars_helper; +use once_cell::sync::Lazy; + +use crate::config::clap::Args; +use crate::config::Config; + +/// 全局的 CONFIG,可以从中读取配置信息 +pub static CONFIG: Lazy = Lazy::new(|| { + let config = Config::load().unwrap_or_else(|err| { + if err + .downcast_ref::() + .map_or(true, |e| e.kind() != std::io::ErrorKind::NotFound) + { + panic!("加载配置文件失败,错误为: {err}"); + } + warn!("配置文件不存在,使用默认配置..."); + Config::default() + }); + // 放到外面,确保新的配置项被保存 + info!("配置加载完毕,覆盖刷新原有配置"); + config.save().unwrap(); + // 检查配置文件内容 + info!("校验配置文件内容..."); + config.check(); + config +}); + +/// 全局的 TEMPLATE,用来渲染 video_name 和 page_name 模板 +pub static TEMPLATE: Lazy = Lazy::new(|| { + let mut handlebars = handlebars::Handlebars::new(); + handlebars_helper!(truncate: |s: String, len: usize| { + if s.chars().count() > len { + s.chars().take(len).collect::() + } else { + s.to_string() + } + }); + handlebars.register_helper("truncate", Box::new(truncate)); + handlebars + .register_template_string("video", &CONFIG.video_name) + .unwrap(); + handlebars.register_template_string("page", &CONFIG.page_name).unwrap(); + handlebars +}); + +/// 全局的 ARGS,用来解析命令行参数 +pub static ARGS: Lazy = Lazy::new(Args::parse); + +/// 全局的 CONFIG_DIR,表示配置文件夹的路径 +pub static CONFIG_DIR: Lazy = + Lazy::new(|| dirs::config_dir().expect("No config path found").join("bili-sync")); diff --git a/crates/bili_sync/src/config/item.rs b/crates/bili_sync/src/config/item.rs new file mode 100644 index 0000000..d973076 --- /dev/null +++ b/crates/bili_sync/src/config/item.rs @@ -0,0 +1,129 @@ +use std::collections::HashMap; +use std::path::PathBuf; + +use serde::de::{Deserializer, MapAccess, Visitor}; +use serde::ser::SerializeMap; +use serde::{Deserialize, Serialize}; + +use crate::bilibili::{CollectionItem, CollectionType}; + +/// 稍后再看的配置 +#[derive(Serialize, Deserialize, Default)] +pub struct WatchLaterConfig { + pub enabled: bool, + pub path: PathBuf, +} + +/// 每次执行操作后的延迟配置 +#[derive(Serialize, Deserialize, Default)] +pub struct DelayConfig { + pub refresh_video_list: Option, + pub fetch_video_detail: Option, + pub download_video: Option, + pub download_page: Option, +} + +/// 延迟的定义,支持固定时间和随机时间 +#[derive(Serialize, Deserialize)] +#[serde(untagged, rename_all = "lowercase")] +pub enum Delay { + Random { min: u64, max: u64 }, + Fixed(u64), +} + +/// NFO 文件使用的时间类型 +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum NFOTimeType { + #[default] + FavTime, + PubTime, +} + +/// 并发下载相关的配置 +#[derive(Serialize, Deserialize)] +pub struct ConcurrentLimit { + pub video: usize, + pub page: usize, + pub delay: DelayConfig, +} + +impl Default for ConcurrentLimit { + fn default() -> Self { + Self { + video: 3, + page: 2, + delay: DelayConfig::default(), + } + } +} + +/* 后面是用于自定义 Collection 的序列化、反序列化的样板代码 */ +pub(super) fn serialize_collection_list( + collection_list: &HashMap, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + let mut map = serializer.serialize_map(Some(collection_list.len()))?; + for (k, v) in collection_list { + let prefix = match k.collection_type { + CollectionType::Series => "series", + CollectionType::Season => "season", + }; + map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?; + } + map.end() +} + +pub(super) fn deserialize_collection_list<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct CollectionListVisitor; + + impl<'de> Visitor<'de> for CollectionListVisitor { + type Value = HashMap; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a map of collection list") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut collection_list = HashMap::new(); + while let Some((key, value)) = map.next_entry::()? { + let collection_item = match key.split(':').collect::>().as_slice() { + [prefix, mid, sid] => { + let collection_type = match *prefix { + "series" => CollectionType::Series, + "season" => CollectionType::Season, + _ => { + return Err(serde::de::Error::custom( + "invalid collection type, should be series or season", + )) + } + }; + CollectionItem { + mid: mid.to_string(), + sid: sid.to_string(), + collection_type, + } + } + _ => { + return Err(serde::de::Error::custom( + "invalid collection key, should be series:mid:sid or season:mid:sid", + )) + } + }; + collection_list.insert(collection_item, value); + } + Ok(collection_list) + } + } + + deserializer.deserialize_map(CollectionListVisitor) +} diff --git a/crates/bili_sync/src/config/mod.rs b/crates/bili_sync/src/config/mod.rs new file mode 100644 index 0000000..b7ddbda --- /dev/null +++ b/crates/bili_sync/src/config/mod.rs @@ -0,0 +1,162 @@ +use std::borrow::Cow; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::Result; +use arc_swap::ArcSwapOption; +use serde::{Deserialize, Serialize}; + +mod clap; +mod global; +mod item; + +use crate::bilibili::{CollectionItem, Credential, DanmakuOption, FilterOption}; +pub use crate::config::global::{ARGS, CONFIG, CONFIG_DIR, TEMPLATE}; +use crate::config::item::{deserialize_collection_list, serialize_collection_list, ConcurrentLimit}; +pub use crate::config::item::{Delay, NFOTimeType, WatchLaterConfig}; + +fn default_time_format() -> String { + "%Y-%m-%d".to_string() +} + +#[derive(Serialize, Deserialize)] +pub struct Config { + pub credential: ArcSwapOption, + pub filter_option: FilterOption, + #[serde(default)] + pub danmaku_option: DanmakuOption, + pub favorite_list: HashMap, + #[serde( + default, + serialize_with = "serialize_collection_list", + deserialize_with = "deserialize_collection_list" + )] + pub collection_list: HashMap, + #[serde(default)] + pub submission_list: HashMap, + #[serde(default)] + pub watch_later: WatchLaterConfig, + pub video_name: Cow<'static, str>, + pub page_name: Cow<'static, str>, + pub interval: u64, + pub upper_path: PathBuf, + #[serde(default)] + pub nfo_time_type: NFOTimeType, + #[serde(default)] + pub concurrent_limit: ConcurrentLimit, + #[serde(default = "default_time_format")] + pub time_format: String, +} + +impl Default for Config { + fn default() -> Self { + Self { + credential: ArcSwapOption::from(Some(Arc::new(Credential::default()))), + filter_option: FilterOption::default(), + danmaku_option: DanmakuOption::default(), + favorite_list: HashMap::new(), + collection_list: HashMap::new(), + submission_list: HashMap::new(), + watch_later: Default::default(), + video_name: Cow::Borrowed("{{title}}"), + page_name: Cow::Borrowed("{{bvid}}"), + interval: 1200, + upper_path: CONFIG_DIR.join("upper_face"), + nfo_time_type: NFOTimeType::FavTime, + concurrent_limit: ConcurrentLimit::default(), + time_format: default_time_format(), + } + } +} + +impl Config { + fn load() -> Result { + let config_path = CONFIG_DIR.join("config.toml"); + let config_content = std::fs::read_to_string(config_path)?; + Ok(toml::from_str(&config_content)?) + } + + pub fn save(&self) -> Result<()> { + let config_path = CONFIG_DIR.join("config.toml"); + std::fs::create_dir_all(&*CONFIG_DIR)?; + std::fs::write(config_path, toml::to_string_pretty(self)?)?; + Ok(()) + } + + pub fn check(&self) { + let mut ok = true; + if self.favorite_list.is_empty() && self.collection_list.is_empty() && !self.watch_later.enabled { + ok = false; + error!("没有配置任何需要扫描的内容,程序空转没有意义"); + } + if self.watch_later.enabled && !self.watch_later.path.is_absolute() { + error!( + "稍后再看保存的路径应为绝对路径,检测到:{}", + self.watch_later.path.display() + ); + } + for path in self.favorite_list.values() { + if !path.is_absolute() { + ok = false; + error!("收藏夹保存的路径应为绝对路径,检测到: {}", path.display()); + } + } + if !self.upper_path.is_absolute() { + ok = false; + error!("up 主头像保存的路径应为绝对路径"); + } + if self.video_name.is_empty() { + ok = false; + error!("未设置 video_name 模板"); + } + if self.page_name.is_empty() { + ok = false; + error!("未设置 page_name 模板"); + } + let credential = self.credential.load(); + match credential.as_deref() { + Some(credential) => { + if credential.sessdata.is_empty() + || credential.bili_jct.is_empty() + || credential.buvid3.is_empty() + || credential.dedeuserid.is_empty() + || credential.ac_time_value.is_empty() + { + ok = false; + error!("Credential 信息不完整,请确保填写完整"); + } + } + None => { + ok = false; + error!("未设置 Credential 信息"); + } + } + if !(self.concurrent_limit.video > 0 && self.concurrent_limit.page > 0) { + ok = false; + error!("允许的并发数必须大于 0"); + } + for delay_config in [ + &self.concurrent_limit.delay.refresh_video_list, + &self.concurrent_limit.delay.fetch_video_detail, + &self.concurrent_limit.delay.download_video, + &self.concurrent_limit.delay.download_page, + ] + .iter() + .filter_map(|x| x.as_ref()) + { + if let Delay::Random { min, max } = delay_config { + if min >= max { + ok = false; + error!("随机延迟的最小值应小于最大值"); + } + } + } + if !ok { + panic!( + "位于 {} 的配置文件不合法,请参考提示信息修复后继续运行", + CONFIG_DIR.join("config.toml").display() + ); + } + } +} diff --git a/crates/bili_sync/src/workflow.rs b/crates/bili_sync/src/workflow.rs index c6f66dc..dc446fd 100644 --- a/crates/bili_sync/src/workflow.rs +++ b/crates/bili_sync/src/workflow.rs @@ -60,7 +60,7 @@ pub async fn refresh_video_list<'a>( info!("到达上一次处理的位置,提前中止"); break; } - delay(CONFIG.delay.refresh_video_list.as_ref()).await; + delay(CONFIG.concurrent_limit.delay.refresh_video_list.as_ref()).await; } new_count = video_list_model.video_count(connection).await? - new_count; video_list_model.log_refresh_video_end(got_count, new_count); @@ -80,7 +80,7 @@ pub async fn fetch_video_details( video_list_model .fetch_videos_detail(video, video_model, connection) .await?; - delay(CONFIG.delay.fetch_video_detail.as_ref()).await; + delay(CONFIG.concurrent_limit.delay.fetch_video_detail.as_ref()).await; } video_list_model.log_fetch_video_end(); Ok(video_list_model) @@ -94,8 +94,7 @@ pub async fn download_unprocessed_videos( ) -> Result<()> { video_list_model.log_download_video_start(); let unhandled_videos_pages = video_list_model.unhandled_video_pages(connection).await?; - // 对于视频,允许三个同时下载(视频内还有分页、不同分页还有多种下载任务) - let semaphore = Semaphore::new(3); + let semaphore = Semaphore::new(CONFIG.concurrent_limit.video); let downloader = Downloader::new(bili_client.client.clone()); let mut uppers_mutex: HashMap, Mutex<()>)> = HashMap::new(); for (video_model, _) in &unhandled_videos_pages { @@ -232,7 +231,7 @@ pub async fn download_video_pages( } let mut video_active_model: video::ActiveModel = video_model.into(); video_active_model.download_status = Set(status.into()); - delay(CONFIG.delay.download_video.as_ref()).await; + delay(CONFIG.concurrent_limit.delay.download_video.as_ref()).await; Ok(video_active_model) } @@ -247,8 +246,7 @@ pub async fn dispatch_download_page( if !should_run { return Ok(()); } - // 对于视频的分页,允许两个同时下载(绝大部分是单页视频) - let child_semaphore = Semaphore::new(2); + let child_semaphore = Semaphore::new(CONFIG.concurrent_limit.page); let mut tasks = pages .into_iter() .map(|page_model| download_page(bili_client, video_model, page_model, &child_semaphore, downloader)) @@ -420,7 +418,7 @@ pub async fn download_page( let mut page_active_model: page::ActiveModel = page_model.into(); page_active_model.download_status = Set(status.into()); page_active_model.path = Set(Some(video_path.to_str().unwrap().to_string())); - delay(CONFIG.delay.download_page.as_ref()).await; + delay(CONFIG.concurrent_limit.delay.download_page.as_ref()).await; Ok(page_active_model) }