From 0b691fc6bd616a7aa83e3a7426492b924e9dbbc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa?= Date: Sun, 19 Nov 2023 22:58:03 +0100 Subject: [PATCH] Don't repeat requests with HTTP, removing www., etc, if the domain is blacklisted --- src/api/icons.rs | 78 +++++++++++++++++++++++++----------------------- src/util.rs | 27 +++++++---------- 2 files changed, 51 insertions(+), 54 deletions(-) diff --git a/src/api/icons.rs b/src/api/icons.rs index ccd4ce451a..ea533a6b49 100644 --- a/src/api/icons.rs +++ b/src/api/icons.rs @@ -156,8 +156,8 @@ pub fn is_domain_blacklisted(domain: &str) -> bool { // If the stored regex is up to date, use it if let Some((value, regex)) = &*guard { - if value == &config_blacklist && regex.is_match(domain) { - return true; + if value == &config_blacklist { + return regex.is_match(domain); } } @@ -166,11 +166,7 @@ pub fn is_domain_blacklisted(domain: &str) -> bool { let is_match = regex.is_match(domain); *guard = Some((config_blacklist, regex)); - if is_match { - return true; - } - - false + is_match } async fn get_icon(domain: &str) -> Option<(Vec, String)> { @@ -356,42 +352,48 @@ async fn get_icon_url(domain: &str) -> Result { let ssldomain = format!("https://{domain}"); let httpdomain = format!("http://{domain}"); - // First check the domain as given during the request for both HTTPS and HTTP. - let resp = match get_page(&ssldomain).or_else(|_| get_page(&httpdomain)).await { - Ok(c) => Ok(c), - Err(e) => { - let mut sub_resp = Err(e); - - // When the domain is not an IP, and has more then one dot, remove all subdomains. - let is_ip = domain.parse::(); - if is_ip.is_err() && domain.matches('.').count() > 1 { - let mut domain_parts = domain.split('.'); - let base_domain = format!( - "{base}.{tld}", - tld = domain_parts.next_back().unwrap(), - base = domain_parts.next_back().unwrap() - ); - if is_valid_domain(&base_domain) { - let sslbase = format!("https://{base_domain}"); - let httpbase = format!("http://{base_domain}"); - debug!("[get_icon_url]: Trying without subdomains '{base_domain}'"); - - sub_resp = get_page(&sslbase).or_else(|_| get_page(&httpbase)).await; - } + // First check the domain as given during the request for HTTPS. + let resp = match get_page(&ssldomain).await { + Err(e) if CustomResolverError::downcast_ref(&e).is_none() => { + // If we get an error that is not caused by the blacklist, we retry with HTTP + match get_page(&httpdomain).await { + mut sub_resp @ Err(_) => { + // When the domain is not an IP, and has more then one dot, remove all subdomains. + let is_ip = domain.parse::(); + if is_ip.is_err() && domain.matches('.').count() > 1 { + let mut domain_parts = domain.split('.'); + let base_domain = format!( + "{base}.{tld}", + tld = domain_parts.next_back().unwrap(), + base = domain_parts.next_back().unwrap() + ); + if is_valid_domain(&base_domain) { + let sslbase = format!("https://{base_domain}"); + let httpbase = format!("http://{base_domain}"); + debug!("[get_icon_url]: Trying without subdomains '{base_domain}'"); + + sub_resp = get_page(&sslbase).or_else(|_| get_page(&httpbase)).await; + } - // When the domain is not an IP, and has less then 2 dots, try to add www. infront of it. - } else if is_ip.is_err() && domain.matches('.').count() < 2 { - let www_domain = format!("www.{domain}"); - if is_valid_domain(&www_domain) { - let sslwww = format!("https://{www_domain}"); - let httpwww = format!("http://{www_domain}"); - debug!("[get_icon_url]: Trying with www. prefix '{www_domain}'"); + // When the domain is not an IP, and has less then 2 dots, try to add www. infront of it. + } else if is_ip.is_err() && domain.matches('.').count() < 2 { + let www_domain = format!("www.{domain}"); + if is_valid_domain(&www_domain) { + let sslwww = format!("https://{www_domain}"); + let httpwww = format!("http://{www_domain}"); + debug!("[get_icon_url]: Trying with www. prefix '{www_domain}'"); - sub_resp = get_page(&sslwww).or_else(|_| get_page(&httpwww)).await; + sub_resp = get_page(&sslwww).or_else(|_| get_page(&httpwww)).await; + } + } + sub_resp } + res => res, } - sub_resp } + + // If we get a result or a blacklist error, just continue + res => res, }; // Create the iconlist diff --git a/src/util.rs b/src/util.rs index 6bcc5ad9b3..a1d75ad37f 100644 --- a/src/util.rs +++ b/src/util.rs @@ -835,16 +835,18 @@ mod dns_resolver { // Note that we get an iterator of addresses, but we only grab the first one for convenience async fn resolve_domain(&self, name: &str) -> Result, BoxError> { - match self { - Self::Default() => { - let mut lookup = tokio::net::lookup_host(name).await?; - Ok(lookup.next()) - } - Self::Hickory(resolver) => { - let lookup = resolver.lookup_ip(name).await?; - Ok(lookup.into_iter().next().map(|a| SocketAddr::new(a, 0))) - } + pre_resolve(name)?; + + let result = match self { + Self::Default() => tokio::net::lookup_host(name).await?.next(), + Self::Hickory(r) => r.lookup_ip(name).await?.iter().next().map(|a| SocketAddr::new(a, 0)), + }; + + if let Some(addr) = &result { + post_resolve(name, addr.ip())?; } + + Ok(result) } } @@ -874,14 +876,7 @@ mod dns_resolver { let this = self.clone(); Box::pin(async move { let name = name.as_str(); - - pre_resolve(name)?; let result = this.resolve_domain(name).await?; - if let Some(addr) = &result { - let ip = addr.ip(); - post_resolve(name, ip)?; - } - Ok::(Box::new(result.into_iter())) }) }