vpnfail scraper added + more output info

build.gradle.kts

@@ -47,22 +47,17 @@ launch4j {
     version = "1.0"
     textVersion = "1.0"
     language = "ENGLISH_UK"
-    //icon = "${projectDir}/icons/myApp.ico"
 }
 
 lateinit var jarFile: File
 
 tasks.withType<Jar> {
     archiveFileName.set("proxy-scraper.jar")
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-    // Otherwise you'll get a "No main manifest attribute" error
     manifest {
         attributes["Main-Class"] = "scraper/Main"
     }
-
-    // To add all of the dependencies otherwise a "NoClassDefFoundError" error
     from(sourceSets.main.get().output)
-
     dependsOn(configurations.runtimeClasspath)
     from({
         configurations.runtimeClasspath.get().filter { it.name.endsWith("jar") }.map { zipTree(it) }

src/main/kotlin/plugin/httpclient/freeproxyapi/FreeProxyApiData.kt

@@ -1,7 +1,13 @@
 package plugin.httpclient.freeproxyapi
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.databind.annotation.JsonSerialize
 
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize
 data class FreeProxyApiData(
     @JsonProperty("Host")
     val host : String,

src/main/kotlin/plugin/httpclient/vpnfail/VpnFail.kt

@@ -0,0 +1,109 @@
+package plugin.httpclient.vpnfail
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.runBlocking
+import mu.KotlinLogging
+import scraper.net.CoroutinesHttpClient
+import scraper.plugin.Plugin
+import scraper.plugin.PluginFactory
+import scraper.plugin.hook.ProxyWebsite
+import scraper.util.NetworkUtils
+import scraper.util.data.ProxyData
+import java.net.http.HttpResponse
+
+/**
+ * GeoNode - 17/02/2023
+ * @author Kai
+ *
+ * Source: https://vpn.fail/
+ * Endpoint: https://vpn.fail/free-proxy/json
+ * Method: GET
+ *
+ * ContentType: JSON
+ * Format:
+ * [
+ *   {
+ *     "proxy": "39.104.62.128:8123",
+ *     "type": "socks5",
+ *   }
+ * ]
+ */
+@Suppress("unused")
+class VpnFail : Plugin, ProxyWebsite {
+
+    private val endpointUrl = "https://vpn.fail/free-proxy/json"
+
+    private val logger = KotlinLogging.logger { }
+
+    override val proxies : MutableList<ProxyData> = mutableListOf()
+
+    private var completed : Boolean = false
+
+    override fun register() {
+        PluginFactory.register(this)
+    }
+
+    override fun initialize() : Boolean {
+        logger.info { "Initializing" }
+
+        try {
+            this.thenConnect()
+        } catch (ex : Exception) {
+            completed = true
+            logger.error { ex.message }
+        }
+
+        return true
+    }
+
+    override fun thenConnect() {
+        logger.info { "Connecting" }
+
+        val response = mutableMapOf<String, HttpResponse<String>?>()
+        runBlocking {
+            val result = async(Dispatchers.IO) {
+                val client = CoroutinesHttpClient()
+                client.contentType = arrayOf("content-type",  "application/json")
+                val data = client.fetch(endpointUrl, null)
+                data
+            }
+            response["json"] = result.await()
+        }
+
+        if (response.isNotEmpty()) {
+            this.thenHandleData(response)
+        } else {
+            logger.error { "Failed to connect to $endpointUrl" }
+        }
+    }
+
+    override fun thenHandleData(data : MutableMap<String, *>) {
+        logger.info { "Handling Data" }
+
+        val mapper = ObjectMapper()
+        val values = mapper.readValue<List<VpnFailData>>((data.getValue("json") as HttpResponse<String>).body())
+        val allowedProtocols = arrayOf("socks4", "socks5", "http", "https")
+
+        for(value in values) {
+            if(!NetworkUtils.isValidIpAndPort(value.host) || !allowedProtocols.contains(value.protocol)) {
+                continue
+            }
+            val ip = value.host.split(":")[0]
+            val port = value.host.split(":")[1]
+            val proxy = ProxyData(ip, port.toInt(), value.protocol)
+            proxies.add(proxy)
+        }
+
+        logger.info { "Collected ${proxies.size} proxies" }
+        completed = true
+        this.finallyComplete()
+    }
+
+    override fun finallyComplete() : Boolean {
+        return completed
+    }
+
+}

src/main/kotlin/plugin/httpclient/vpnfail/VpnFailData.kt

@@ -0,0 +1,16 @@
+package plugin.httpclient.vpnfail
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonInclude
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.databind.annotation.JsonSerialize
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize
+data class VpnFailData(
+    @JsonProperty("proxy")
+    val host : String,
+    @JsonProperty("type")
+    val protocol : String
+)

src/main/kotlin/scraper/Main.kt

@@ -87,6 +87,10 @@ class Main {
 
         val jsonFile = Path.of("$outputPath/proxies.json").toFile()
         mapper.writeValue(jsonFile, proxyList)
+
+        logger.info { "Total unique proxies collected: ${allProxies.size}" }
+        logger.info { "HTTP:[${proxyList.http.size}] | HTTPS:[${proxyList.https.size}] " +
+                "| SOCKS4:[${proxyList.socks4.size}] | SOCKS5:[${proxyList.socks5.size}]" }
     }
 
 }