From 5b9698504df053a8b50e67b6579d9b469ab07abf Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+SamTV12345@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:38:51 +0200 Subject: [PATCH] Added new proxy var PODFETCH_PROXY (#904) Co-authored-by: Samuel Schwanzer --- docs/src/HOSTING.md | 7 +++- src/controllers/podcast_controller.rs | 16 ++++----- src/main.rs | 8 ++++- src/service/podcast_episode_service.rs | 4 +-- src/service/rust_service.rs | 6 ++-- src/utils/mod.rs | 1 + src/utils/reqwest_client.rs | 45 ++++++++++++++++++++++++++ 7 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 src/utils/reqwest_client.rs diff --git a/docs/src/HOSTING.md b/docs/src/HOSTING.md index 5583fb0c..0c2518ab 100644 --- a/docs/src/HOSTING.md +++ b/docs/src/HOSTING.md @@ -25,4 +25,9 @@ To enable it you need to set the following variables: You can acquire the Telegram Bot chat id with the following steps: 1. Write a message to the bot 2. Open the following url in your browser: [Telegram API page](https://api.telegram.org/bot/getUpdates) -3. Search for the chat id in the response \ No newline at end of file +3. Search for the chat id in the response + + +# Proxying requests to the podcast servers + +In some cases you also need to proxy the traffic from the PodFetch server via a proxy. For that exists the `PODFETCH_PROXY` variable. You set it to the address of your proxy. \ No newline at end of file diff --git a/src/controllers/podcast_controller.rs b/src/controllers/podcast_controller.rs index 87134e85..f2f48f2e 100644 --- a/src/controllers/podcast_controller.rs +++ b/src/controllers/podcast_controller.rs @@ -22,8 +22,6 @@ use futures::executor; use opml::{Outline, OPML}; use rand::rngs::ThreadRng; use rand::Rng; -use reqwest::blocking::{Client, ClientBuilder as SyncClientBuilder}; -use reqwest::{ClientBuilder as AsyncClientBuilder}; use rss::Channel; use serde_json::{from_str, Value}; use std::ops::DerefMut; @@ -43,6 +41,7 @@ use crate::service::file_service::{perform_podcast_variable_replacement, FileSer use crate::utils::append_to_header::add_basic_auth_headers_conditionally; use crate::DBType as DbConnection; use futures_util::StreamExt; +use reqwest::Client; use reqwest::header::HeaderMap; use tokio::sync::mpsc; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -247,7 +246,7 @@ pub async fn add_podcast( if !requester.unwrap().is_privileged_user() { return Err(CustomError::Forbidden); } - let client = AsyncClientBuilder::new().build().unwrap(); + let client = get_async_sync_client().build().unwrap(); let query: Vec<(&str, String)> = vec![ ("id", track_id.track_id.to_string()), @@ -299,7 +298,7 @@ pub async fn add_podcast_by_feed( if !requester.unwrap().is_privileged_user() { return Err(CustomError::Forbidden); } - let client = AsyncClientBuilder::new().build().unwrap(); + let client = get_async_sync_client().build().unwrap(); let mut header_map = HeaderMap::new(); header_map.insert("User-Agent", COMMON_USER_AGENT.parse().unwrap()); add_basic_auth_headers_conditionally(rss_feed.clone().rss_feed_url, &mut header_map); @@ -364,7 +363,7 @@ pub async fn import_podcasts_from_opml( let document = OPML::from_str(&opml.content).unwrap(); for outline in document.body.outlines { - let client = SyncClientBuilder::new().build().unwrap(); + let client = get_async_sync_client().build().unwrap(); executor::block_on(insert_outline( outline.clone(), client.clone(), @@ -646,7 +645,7 @@ async fn insert_outline( return; } - let feed_response = client.get(feed_url.unwrap()).send(); + let feed_response = client.get(feed_url.unwrap()).send().await; if feed_response.is_err() { lobby.do_send(BroadcastMessage { type_of: PodcastType::OpmlErrored, @@ -657,7 +656,7 @@ async fn insert_outline( }); return; } - let content = feed_response.unwrap().bytes().unwrap(); + let content = feed_response.unwrap().bytes().await.unwrap(); let channel = Channel::read_from(&content[..]); @@ -725,6 +724,7 @@ use crate::models::settings::Setting; use crate::utils::environment_variables::is_env_var_present_and_true; use crate::utils::error::{map_r2d2_error, map_reqwest_error, CustomError}; +use crate::utils::reqwest_client::get_async_sync_client; use crate::utils::rss_feed_parser::PodcastParsed; #[derive(Deserialize, ToSchema)] @@ -832,7 +832,7 @@ pub(crate) async fn proxy_podcast( header_map.append("sec-fetch-mode", "no-cors".parse().unwrap()); header_map.append("sec-fetch-site", "cross-site".parse().unwrap()); use std::str::FromStr; - let forwarded_req = reqwest::Client::new() + let forwarded_req = get_async_sync_client().build().unwrap() .request(reqwest::Method::from_str(method.as_str()).unwrap(), episode.url) .headers(header_map) .fetch_mode_no_cors() diff --git a/src/main.rs b/src/main.rs index 2e4b6c17..bde13d69 100644 --- a/src/main.rs +++ b/src/main.rs @@ -92,6 +92,7 @@ use crate::service::rust_service::PodcastService; use crate::service::settings_service::SettingsService; use crate::utils::error::CustomError; use crate::utils::podcast_key_checker::check_podcast_request; +use crate::utils::reqwest_client::get_async_sync_client; mod config; @@ -277,7 +278,12 @@ async fn main() -> std::io::Result<()> { match ENVIRONMENT_SERVICE.get().unwrap().oidc_config.clone() { Some(jwk_config) => { - let resp = reqwest::get(&jwk_config.jwks_uri) + let client = get_async_sync_client() + .build() + .unwrap(); + let resp = client + .get(&jwk_config.jwks_uri) + .send() .await .unwrap() .json::() diff --git a/src/service/podcast_episode_service.rs b/src/service/podcast_episode_service.rs index 63e81588..bb8543ef 100644 --- a/src/service/podcast_episode_service.rs +++ b/src/service/podcast_episode_service.rs @@ -19,7 +19,6 @@ use actix_web::web; use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use log::error; use regex::Regex; -use reqwest::blocking::ClientBuilder; use reqwest::header::{HeaderMap, ACCEPT}; use reqwest::redirect::Policy; use rss::{Channel, Item}; @@ -35,6 +34,7 @@ use crate::service::settings_service::SettingsService; use crate::service::telegram_api::send_new_episode_notification; use crate::utils::environment_variables::is_env_var_present_and_true; use crate::utils::error::{map_db_error, CustomError}; +use crate::utils::reqwest_client::get_sync_client; pub struct PodcastEpisodeService; @@ -566,7 +566,7 @@ impl PodcastEpisodeService { fn do_request_to_podcast_server(podcast: Podcast) -> RequestReturnType { let is_redirected = Arc::new(Mutex::new(false)); // Variable to store the redirection status - let client = ClientBuilder::new() + let client = get_sync_client() .redirect(Policy::custom({ let is_redirected = Arc::clone(&is_redirected); diff --git a/src/service/rust_service.rs b/src/service/rust_service.rs index daf029b0..63b08f4a 100644 --- a/src/service/rust_service.rs +++ b/src/service/rust_service.rs @@ -15,12 +15,11 @@ use crate::unwrap_string; use actix::Addr; use actix_web::web::Data; use reqwest::header::{HeaderMap, HeaderValue}; -use reqwest::{Client, ClientBuilder as AsyncClientBuilder}; use rss::Channel; use serde_json::Value; use sha1::{Digest, Sha1}; use std::time::SystemTime; - +use reqwest::Client; use crate::config::dbconfig::establish_connection; use serde::Serialize; use tokio::task::spawn_blocking; @@ -31,6 +30,7 @@ use crate::models::settings::Setting; use crate::utils::error::{map_reqwest_error, CustomError}; use crate::DBType as DbConnection; use crate::models::podcast_settings::PodcastSetting; +use crate::utils::reqwest_client::get_async_sync_client; #[derive(Clone)] pub struct PodcastService { @@ -46,7 +46,7 @@ impl Default for PodcastService { impl PodcastService { pub fn new() -> PodcastService { PodcastService { - client: AsyncClientBuilder::new().build().unwrap(), + client: get_async_sync_client().build().unwrap(), } } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 8da23804..d6e180ab 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -7,3 +7,4 @@ pub mod podcast_builder; pub mod podcast_key_checker; pub(crate) mod rss_feed_parser; pub mod time; +pub mod reqwest_client; diff --git a/src/utils/reqwest_client.rs b/src/utils/reqwest_client.rs new file mode 100644 index 00000000..ec89c743 --- /dev/null +++ b/src/utils/reqwest_client.rs @@ -0,0 +1,45 @@ +use std::env; +use reqwest::blocking::ClientBuilder; +use reqwest::Proxy; + +pub fn get_sync_client() -> ClientBuilder { + let proxy_val = env::var("PODFETCH_PROXY"); + + + + let mut res = ClientBuilder::new(); + + if let Ok(unwrapped_proxy) = proxy_val { + let proxy = Proxy::all(unwrapped_proxy); + match proxy { + Ok(e)=>{ + res = res.proxy(e); + } + Err(e)=>{ + log::error!("Proxy is invalid {}", e) + } + } + } + + res +} + +pub fn get_async_sync_client() -> reqwest::ClientBuilder { + let proxy_val = env::var("PODFETCH_PROXY"); + + let mut res = reqwest::ClientBuilder::new(); + + if let Ok(unwrapped_proxy) = proxy_val { + let proxy = Proxy::all(unwrapped_proxy); + match proxy { + Ok(e)=>{ + res = res.proxy(e); + } + Err(e)=>{ + log::error!("Proxy is invalid {}", e) + } + } + } + + res +} \ No newline at end of file