diff --git a/src/main/kotlin/plugin/httpclient/checkerproxy/CheckerProxy.kt b/src/main/kotlin/plugin/httpclient/checkerproxy/CheckerProxy.kt index 4585afa..7e4c018 100644 --- a/src/main/kotlin/plugin/httpclient/checkerproxy/CheckerProxy.kt +++ b/src/main/kotlin/plugin/httpclient/checkerproxy/CheckerProxy.kt @@ -54,7 +54,7 @@ class CheckerProxy : Plugin, ProxyWebsite { logger.error { ex.message } } - return false + return true } override fun thenConnect() { diff --git a/src/main/kotlin/plugin/httpclient/freeproxyapi/FreeProxyApi.kt b/src/main/kotlin/plugin/httpclient/freeproxyapi/FreeProxyApi.kt index 34ef2d9..848ae18 100644 --- a/src/main/kotlin/plugin/httpclient/freeproxyapi/FreeProxyApi.kt +++ b/src/main/kotlin/plugin/httpclient/freeproxyapi/FreeProxyApi.kt @@ -58,7 +58,7 @@ class FreeProxyApi : Plugin, ProxyWebsite { logger.error { ex.message } } - return false + return true } override fun thenConnect() { diff --git a/src/main/kotlin/plugin/httpclient/geonode/GeoNode.kt b/src/main/kotlin/plugin/httpclient/geonode/GeoNode.kt index 9444aca..e25ba34 100644 --- a/src/main/kotlin/plugin/httpclient/geonode/GeoNode.kt +++ b/src/main/kotlin/plugin/httpclient/geonode/GeoNode.kt @@ -91,7 +91,7 @@ class GeoNode : Plugin, ProxyWebsite { logger.error { ex.message } } - return false + return true } override fun thenConnect() { diff --git a/src/main/kotlin/plugin/httpclient/openproxylist/OpenProxyList.kt b/src/main/kotlin/plugin/httpclient/openproxylist/OpenProxyList.kt index dc1f390..a13d18a 100644 --- a/src/main/kotlin/plugin/httpclient/openproxylist/OpenProxyList.kt +++ b/src/main/kotlin/plugin/httpclient/openproxylist/OpenProxyList.kt @@ -57,7 +57,7 @@ class OpenProxyList : Plugin, ProxyWebsite { logger.error { ex.message } } - return false + return true } override fun thenConnect() { diff --git a/src/main/kotlin/plugin/httpclient/proxyscrape/ProxyScrape.kt b/src/main/kotlin/plugin/httpclient/proxyscrape/ProxyScrape.kt index 7e53a98..fa3b7ab 100644 --- a/src/main/kotlin/plugin/httpclient/proxyscrape/ProxyScrape.kt +++ b/src/main/kotlin/plugin/httpclient/proxyscrape/ProxyScrape.kt @@ -59,7 +59,7 @@ class ProxyScrape : Plugin, ProxyWebsite { logger.error { ex.message } } - return false + return true } override fun thenConnect() { diff --git a/src/main/kotlin/scraper/Main.kt b/src/main/kotlin/scraper/Main.kt index 0293b2c..810106e 100644 --- a/src/main/kotlin/scraper/Main.kt +++ b/src/main/kotlin/scraper/Main.kt @@ -10,6 +10,12 @@ import java.nio.file.Path import kotlin.io.path.exists import kotlin.system.exitProcess +/** + * Main - 12/02/2023 + * @author Kai + * + * Description: Main class, runs the application + **/ class Main { private val logger = KotlinLogging.logger { } diff --git a/src/main/kotlin/scraper/net/ChromeWebDriver.kt b/src/main/kotlin/scraper/net/ChromeWebDriver.kt index 49a08bc..b26e3d4 100644 --- a/src/main/kotlin/scraper/net/ChromeWebDriver.kt +++ b/src/main/kotlin/scraper/net/ChromeWebDriver.kt @@ -12,6 +12,12 @@ import org.openqa.selenium.devtools.Command import org.openqa.selenium.devtools.DevTools import org.openqa.selenium.devtools.v109.network.Network +/** + * ChromeWebDriver - 12/02/2023 + * @author Kai + * + * Description: Used for more complex proxy sites, ones that require JavaScript & obfuscate their data + **/ class ChromeWebDriver { private val options = ChromeOptions() diff --git a/src/main/kotlin/scraper/net/CoroutinesHttpClient.kt b/src/main/kotlin/scraper/net/CoroutinesHttpClient.kt index 190992d..d0cfa03 100644 --- a/src/main/kotlin/scraper/net/CoroutinesHttpClient.kt +++ b/src/main/kotlin/scraper/net/CoroutinesHttpClient.kt @@ -9,14 +9,21 @@ import java.net.http.HttpResponse import java.net.http.HttpTimeoutException import java.time.Duration - +/** + * CoroutinesHttpClient - 12/02/2023 + * @author Kai + * + * Description: Used for simple web calls, such as sending GET/POST data for text content (JSON etc) + **/ class CoroutinesHttpClient { companion object { + val timeout : Duration = Duration.ofSeconds(15) + private val httpClient : HttpClient = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(30)) + .version(HttpClient.Version.HTTP_2) + .connectTimeout(timeout) .build() } @@ -31,7 +38,7 @@ class CoroutinesHttpClient { val request : HttpRequest = if(postData == null) { HttpRequest.newBuilder() .uri(URI.create(url)) - .timeout(Duration.ofSeconds(30)) + .timeout(timeout) .headers(*userAgent) .headers(*accept) .headers(*acceptLanguage) @@ -42,7 +49,7 @@ class CoroutinesHttpClient { } else { HttpRequest.newBuilder() .uri(URI.create(url)) - .timeout(Duration.ofSeconds(30)) + .timeout(timeout) .headers(*userAgent) .headers(*accept) .headers(*acceptLanguage) diff --git a/src/main/kotlin/scraper/plugin/Plugin.kt b/src/main/kotlin/scraper/plugin/Plugin.kt index c193e36..5cafb0e 100644 --- a/src/main/kotlin/scraper/plugin/Plugin.kt +++ b/src/main/kotlin/scraper/plugin/Plugin.kt @@ -1,5 +1,11 @@ package scraper.plugin +/** + * Plugin - 12/02/2023 + * @author Kai + * + * Description: Interface for Plugin, this will append the class to a List within PluginFactory + **/ interface Plugin { fun register() diff --git a/src/main/kotlin/scraper/plugin/PluginFactory.kt b/src/main/kotlin/scraper/plugin/PluginFactory.kt index 67d7bc5..8942d49 100644 --- a/src/main/kotlin/scraper/plugin/PluginFactory.kt +++ b/src/main/kotlin/scraper/plugin/PluginFactory.kt @@ -5,6 +5,13 @@ import org.reflections.Reflections import scraper.plugin.hook.ProxyWebsite import java.lang.reflect.Modifier +/** + * PluginFactory - 12/02/2023 + * @author Kai + * + * Description: This is used for loading the Proxy Website Plugins from /src/main/kotlin/plugins/ + * Makes it easier for APIs to be added / removed + **/ object PluginFactory { private val logger = KotlinLogging.logger { } diff --git a/src/main/kotlin/scraper/plugin/hook/ProxyWebsite.kt b/src/main/kotlin/scraper/plugin/hook/ProxyWebsite.kt index db7882f..a3d52d7 100644 --- a/src/main/kotlin/scraper/plugin/hook/ProxyWebsite.kt +++ b/src/main/kotlin/scraper/plugin/hook/ProxyWebsite.kt @@ -2,6 +2,13 @@ package scraper.plugin.hook import scraper.util.data.ProxyData +/** + * ProxyWebsite - 12/02/2023 + * @author Kai + * + * Description: Interface for the Website Plugins within /src/main/kotlin/plugin/ + * Plugins will follow this interface format + **/ interface ProxyWebsite { val proxies : MutableList diff --git a/src/main/kotlin/scraper/util/NetworkUtils.kt b/src/main/kotlin/scraper/util/NetworkUtils.kt index dbb87da..db2360c 100644 --- a/src/main/kotlin/scraper/util/NetworkUtils.kt +++ b/src/main/kotlin/scraper/util/NetworkUtils.kt @@ -2,6 +2,12 @@ package scraper.util import java.util.regex.Pattern +/** + * NetworkUtils - 17/02/2023 + * @author Kai + * + * Description: Any reusable network functions are in this class + **/ object NetworkUtils { fun isValidIpAddress(ipAddress : String) : Boolean { diff --git a/src/main/kotlin/scraper/util/ScraperExecutor.kt b/src/main/kotlin/scraper/util/ScraperExecutor.kt index d9e2df4..908d635 100644 --- a/src/main/kotlin/scraper/util/ScraperExecutor.kt +++ b/src/main/kotlin/scraper/util/ScraperExecutor.kt @@ -3,6 +3,12 @@ package scraper.util import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +/** + * ScraperExecutor - 17/02/2023 + * @author Kai + * + * Description: This is used for multi-threading to speed up the application + **/ object ScraperExecutor { private val PROCESSORS = Runtime.getRuntime().availableProcessors() + 1 diff --git a/src/main/kotlin/scraper/util/ScraperThreadFactory.kt b/src/main/kotlin/scraper/util/ScraperThreadFactory.kt index 90a5f21..7e74430 100644 --- a/src/main/kotlin/scraper/util/ScraperThreadFactory.kt +++ b/src/main/kotlin/scraper/util/ScraperThreadFactory.kt @@ -3,6 +3,12 @@ package scraper.util import java.util.concurrent.ThreadFactory import java.util.concurrent.atomic.AtomicInteger +/** + * ScraperThreadFactory - 12/02/2023 + * @author Kai + * + * Description: This is used for multi-threading to speed up the application + **/ class ScraperThreadFactory(private val name : String) : ThreadFactory { private val threadCount = AtomicInteger() override fun newThread(r : Runnable): Thread { diff --git a/src/main/kotlin/scraper/util/scripts/GetAllCheckerProxies.kt b/src/main/kotlin/scraper/util/scripts/GetAllCheckerProxies.kt index afc4735..ad51d45 100644 --- a/src/main/kotlin/scraper/util/scripts/GetAllCheckerProxies.kt +++ b/src/main/kotlin/scraper/util/scripts/GetAllCheckerProxies.kt @@ -18,6 +18,12 @@ import java.time.format.DateTimeFormatter import kotlin.io.path.exists import kotlin.system.exitProcess +/** + * GetAllCheckerProxies - 17/02/2023 + * @author Kai + * + * Description: Quick and easy script for getting all the proxies from CheckerProxy's API + **/ private const val endpoint = "https://checkerproxy.net/api/archive/" private val proxies : MutableList = mutableListOf() @@ -26,7 +32,7 @@ fun main() { loopThroughDates() } -fun loopThroughDates() { +private fun loopThroughDates() { val format = DateTimeFormatter.ofPattern("yyyy-MM-dd") val fromDate = LocalDate.parse("2023-01-18", format) val toDate = LocalDate.parse("2023-02-17", format)