Added new proxy var PODFETCH_PROXY (#904)

Co-authored-by: Samuel Schwanzer <[email protected]>

docs/src/HOSTING.md

@@ -25,4 +25,9 @@ To enable it you need to set the following variables:
 You can acquire the Telegram Bot chat id with the following steps:
 1. Write a message to the bot
 2. Open the following url in your browser: [Telegram API page](https://api.telegram.org/bot<TELEGRAM_BOT_TOKEN>/getUpdates)
-3. Search for the chat id in the response
+3. Search for the chat id in the response
+
+
+# Proxying requests to the podcast servers
+
+In some cases you also need to proxy the traffic from the PodFetch server via a proxy. For that exists the `PODFETCH_PROXY` variable. You set it to the address of your proxy.

src/controllers/podcast_controller.rs

@@ -22,8 +22,6 @@ use futures::executor;
 use opml::{Outline, OPML};
 use rand::rngs::ThreadRng;
 use rand::Rng;
-use reqwest::blocking::{Client, ClientBuilder as SyncClientBuilder};
-use reqwest::{ClientBuilder as AsyncClientBuilder};
 use rss::Channel;
 use serde_json::{from_str, Value};
 use std::ops::DerefMut;
@@ -43,6 +41,7 @@ use crate::service::file_service::{perform_podcast_variable_replacement, FileSer
 use crate::utils::append_to_header::add_basic_auth_headers_conditionally;
 use crate::DBType as DbConnection;
 use futures_util::StreamExt;
+use reqwest::Client;
 use reqwest::header::HeaderMap;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::UnboundedReceiverStream;
@@ -247,7 +246,7 @@ pub async fn add_podcast(
     if !requester.unwrap().is_privileged_user() {
         return Err(CustomError::Forbidden);
     }
-    let client = AsyncClientBuilder::new().build().unwrap();
+    let client = get_async_sync_client().build().unwrap();
 
     let query: Vec<(&str, String)> = vec![
         ("id", track_id.track_id.to_string()),
@@ -299,7 +298,7 @@ pub async fn add_podcast_by_feed(
     if !requester.unwrap().is_privileged_user() {
         return Err(CustomError::Forbidden);
     }
-    let client = AsyncClientBuilder::new().build().unwrap();
+    let client = get_async_sync_client().build().unwrap();
     let mut header_map = HeaderMap::new();
     header_map.insert("User-Agent", COMMON_USER_AGENT.parse().unwrap());
     add_basic_auth_headers_conditionally(rss_feed.clone().rss_feed_url, &mut header_map);
@@ -364,7 +363,7 @@ pub async fn import_podcasts_from_opml(
         let document = OPML::from_str(&opml.content).unwrap();
 
         for outline in document.body.outlines {
-            let client = SyncClientBuilder::new().build().unwrap();
+            let client = get_async_sync_client().build().unwrap();
             executor::block_on(insert_outline(
                 outline.clone(),
                 client.clone(),
@@ -646,7 +645,7 @@ async fn insert_outline(
         return;
     }
 
-    let feed_response = client.get(feed_url.unwrap()).send();
+    let feed_response = client.get(feed_url.unwrap()).send().await;
     if feed_response.is_err() {
         lobby.do_send(BroadcastMessage {
             type_of: PodcastType::OpmlErrored,
@@ -657,7 +656,7 @@ async fn insert_outline(
         });
         return;
     }
-    let content = feed_response.unwrap().bytes().unwrap();
+    let content = feed_response.unwrap().bytes().await.unwrap();
 
     let channel = Channel::read_from(&content[..]);
 
@@ -725,6 +724,7 @@ use crate::models::settings::Setting;
 use crate::utils::environment_variables::is_env_var_present_and_true;
 
 use crate::utils::error::{map_r2d2_error, map_reqwest_error, CustomError};
+use crate::utils::reqwest_client::get_async_sync_client;
 use crate::utils::rss_feed_parser::PodcastParsed;
 
 #[derive(Deserialize, ToSchema)]
@@ -832,7 +832,7 @@ pub(crate) async fn proxy_podcast(
     header_map.append("sec-fetch-mode", "no-cors".parse().unwrap());
     header_map.append("sec-fetch-site", "cross-site".parse().unwrap());
     use std::str::FromStr;
-    let forwarded_req = reqwest::Client::new()
+    let forwarded_req = get_async_sync_client().build().unwrap()
         .request(reqwest::Method::from_str(method.as_str()).unwrap(), episode.url)
         .headers(header_map)
         .fetch_mode_no_cors()

src/main.rs

@@ -92,6 +92,7 @@ use crate::service::rust_service::PodcastService;
 use crate::service::settings_service::SettingsService;
 use crate::utils::error::CustomError;
 use crate::utils::podcast_key_checker::check_podcast_request;
+use crate::utils::reqwest_client::get_async_sync_client;
 
 mod config;
 
@@ -277,7 +278,12 @@ async fn main() -> std::io::Result<()> {
 
     match ENVIRONMENT_SERVICE.get().unwrap().oidc_config.clone() {
         Some(jwk_config) => {
-            let resp = reqwest::get(&jwk_config.jwks_uri)
+            let client = get_async_sync_client()
+                .build()
+                .unwrap();
+            let resp = client
+                .get(&jwk_config.jwks_uri)
+                .send()
                 .await
                 .unwrap()
                 .json::<CustomJwkSet>()

src/service/podcast_episode_service.rs

@@ -19,7 +19,6 @@ use actix_web::web;
 use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl};
 use log::error;
 use regex::Regex;
-use reqwest::blocking::ClientBuilder;
 use reqwest::header::{HeaderMap, ACCEPT};
 use reqwest::redirect::Policy;
 use rss::{Channel, Item};
@@ -35,6 +34,7 @@ use crate::service::settings_service::SettingsService;
 use crate::service::telegram_api::send_new_episode_notification;
 use crate::utils::environment_variables::is_env_var_present_and_true;
 use crate::utils::error::{map_db_error, CustomError};
+use crate::utils::reqwest_client::get_sync_client;
 
 pub struct PodcastEpisodeService;
 
@@ -566,7 +566,7 @@ impl PodcastEpisodeService {
 
     fn do_request_to_podcast_server(podcast: Podcast) -> RequestReturnType {
         let is_redirected = Arc::new(Mutex::new(false)); // Variable to store the redirection status
-        let client = ClientBuilder::new()
+        let client = get_sync_client()
             .redirect(Policy::custom({
                 let is_redirected = Arc::clone(&is_redirected);
 

src/service/rust_service.rs

@@ -15,12 +15,11 @@ use crate::unwrap_string;
 use actix::Addr;
 use actix_web::web::Data;
 use reqwest::header::{HeaderMap, HeaderValue};
-use reqwest::{Client, ClientBuilder as AsyncClientBuilder};
 use rss::Channel;
 use serde_json::Value;
 use sha1::{Digest, Sha1};
 use std::time::SystemTime;
-
+use reqwest::Client;
 use crate::config::dbconfig::establish_connection;
 use serde::Serialize;
 use tokio::task::spawn_blocking;
@@ -31,6 +30,7 @@ use crate::models::settings::Setting;
 use crate::utils::error::{map_reqwest_error, CustomError};
 use crate::DBType as DbConnection;
 use crate::models::podcast_settings::PodcastSetting;
+use crate::utils::reqwest_client::get_async_sync_client;
 
 #[derive(Clone)]
 pub struct PodcastService {
@@ -46,7 +46,7 @@ impl Default for PodcastService {
 impl PodcastService {
     pub fn new() -> PodcastService {
         PodcastService {
-            client: AsyncClientBuilder::new().build().unwrap(),
+            client: get_async_sync_client().build().unwrap(),
         }
     }
 

src/utils/mod.rs

@@ -7,3 +7,4 @@ pub mod podcast_builder;
 pub mod podcast_key_checker;
 pub(crate) mod rss_feed_parser;
 pub mod time;
+pub mod reqwest_client;

src/utils/reqwest_client.rs

@@ -0,0 +1,45 @@
+use std::env;
+use reqwest::blocking::ClientBuilder;
+use reqwest::Proxy;
+
+pub fn get_sync_client() -> ClientBuilder {
+    let proxy_val  = env::var("PODFETCH_PROXY");
+
+
+
+    let mut res = ClientBuilder::new();
+
+    if let Ok(unwrapped_proxy) = proxy_val {
+        let proxy = Proxy::all(unwrapped_proxy);
+        match proxy {
+            Ok(e)=>{
+                res = res.proxy(e);
+            }
+            Err(e)=>{
+                log::error!("Proxy is invalid {}", e)
+            }
+        }
+    }
+
+    res
+}
+
+pub fn get_async_sync_client() -> reqwest::ClientBuilder {
+    let proxy_val  = env::var("PODFETCH_PROXY");
+
+    let mut res = reqwest::ClientBuilder::new();
+
+    if let Ok(unwrapped_proxy) = proxy_val {
+        let proxy = Proxy::all(unwrapped_proxy);
+        match proxy {
+            Ok(e)=>{
+                res = res.proxy(e);
+            }
+            Err(e)=>{
+                log::error!("Proxy is invalid {}", e)
+            }
+        }
+    }
+
+    res
+}