Merge branch 'feat/ai'

.gitignore

@@ -159,3 +159,5 @@ gradle-app.setting
 /wai2k/
 
 .idea/modules.xml
+
+assets/models/

build.gradle.kts

@@ -23,6 +23,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import java.nio.file.Files
 import java.nio.file.Paths
 import java.nio.file.StandardOpenOption
+import java.security.MessageDigest
+import javax.xml.bind.DatatypeConverter
 
 plugins {
     java
@@ -60,6 +62,8 @@ dependencies {
     implementation("org.controlsfx", "controlsfx", "8.40.14")
     implementation("org.reflections", "reflections", "0.9.12")
     implementation("com.squareup.okhttp3:okhttp:4.7.2")
+    implementation("ai.djl.pytorch:pytorch-engine:0.8.0")
+    implementation("ai.djl.pytorch:pytorch-native-auto:1.6.0")
 
     implementation("net.sourceforge.tess4j", "tess4j", "4.5.1") {
         exclude("org.ghost4j")
@@ -98,11 +102,16 @@ tasks {
         dependencies {
             include { it.moduleGroup.startsWith("com.waicool20") }
         }
+        doLast { md5sum(archiveFile.get()) }
     }
 }
 
+task("prepare-deploy") {
+    dependsOn("build", "deps-list", "packAssets")
+}
+
 task("deps-list") {
-    val file = Paths.get("$projectDir/build/dependencies.txt")
+    val file = Paths.get("$buildDir/deploy/dependencies.txt")
     doFirst {
         if (Files.notExists(file)) {
             Files.createDirectories(file.parent)
@@ -141,7 +150,28 @@ task("versioning") {
 
 task<Zip>("packAssets") {
     archiveFileName.set("assets.zip")
-    destinationDirectory.set(file("$buildDir"))
+    destinationDirectory.set(file("$buildDir/deploy/"))
+
+    from(projectDir)
+    include("/assets/**")
+    exclude("/assets/models/**")
+    doLast { md5sum(archiveFile.get()) }
+}
+
+task<Zip>("packModels") {
+    archiveFileName.set("models.zip")
+    destinationDirectory.set(file("$buildDir/deploy/"))
+
+    from(projectDir)
+    include("/assets/models/**")
+    doLast { md5sum(archiveFile.get()) }
+}
 
-    from("$projectDir/assets")
+fun md5sum(file: RegularFile) {
+    val path = file.asFile.toPath()
+    val md5File = Paths.get("$path.md5")
+    val md5sum = MessageDigest.getInstance("MD5")
+        .digest(Files.readAllBytes(path))
+        .let { DatatypeConverter.printHexBinary(it) }
+    Files.write(md5File, md5sum.toByteArray())
 }

launcher/build.gradle.kts

@@ -25,6 +25,10 @@
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import org.jetbrains.kotlin.gradle.plugin.KotlinPluginWrapper
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.security.MessageDigest
+import javax.xml.bind.DatatypeConverter
 
 plugins {
     java
@@ -68,5 +72,15 @@ tasks {
         archiveClassifier.value("")
         archiveVersion.value("")
         exclude("kotlin/reflect/**")
+        doLast { md5sum(archiveFile.get()) }
     }
 }
+
+fun md5sum(file: RegularFile) {
+    val path = file.asFile.toPath()
+    val md5File = Paths.get("$path.md5")
+    val md5sum = MessageDigest.getInstance("MD5")
+        .digest(Files.readAllBytes(path))
+        .let { DatatypeConverter.printHexBinary(it) }
+    Files.write(md5File, md5sum.toByteArray())
+}

src/main/kotlin/com/waicool20/wai2k/Wai2K.kt

@@ -48,6 +48,9 @@ class Wai2K : App(Wai2KWorkspace::class) {
             } catch (t: Throwable) {
                 logger.warn("Could not set locale to C, application may crash if using tesseract 4.0+")
             }
+            // Set inference thread count, anything above 8 seems to be ignored
+            val cores = Runtime.getRuntime().availableProcessors().coerceAtMost(8)
+            CLib.setEnv("OMP_NUM_THREADS", cores.toString(), true)
         }
 
         private fun isRunningJar(): Boolean {

src/main/kotlin/com/waicool20/wai2k/script/modules/CombatSimModule.kt

@@ -64,7 +64,7 @@ class CombatSimModule(
 
     private val mapRunner = object : AbsoluteMapRunner(scriptRunner, region, config, profile) {
         override val isCorpseDraggingMap = false
-        override suspend fun execute() {
+        override suspend fun begin() {
             if (profile.combatSimulation.neuralFragment == Level.OFF) return
 
             if (OffsetDateTime.now(ZoneOffset.ofHours(-8)).dayOfWeek !in dataSimDays) {

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/HomographyMapRunner.kt

@@ -19,22 +19,25 @@
 
 package com.waicool20.wai2k.script.modules.combat
 
-import boofcv.struct.image.GrayF32
+import ai.djl.metric.Metrics
+import ai.djl.modality.cv.Image
+import ai.djl.modality.cv.ImageFactory
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.waicool20.cvauto.android.AndroidRegion
-import com.waicool20.cvauto.util.homography
 import com.waicool20.cvauto.util.transformRect
 import com.waicool20.wai2k.config.Wai2KConfig
 import com.waicool20.wai2k.config.Wai2KProfile
-import com.waicool20.wai2k.script.NodeNotFoundException
 import com.waicool20.wai2k.script.ScriptRunner
-import com.waicool20.wai2k.util.extractNodes
+import com.waicool20.wai2k.util.ai.MatchingModel
+import com.waicool20.wai2k.util.ai.MatchingTranslator
 import com.waicool20.waicoolutils.logging.loggerFor
 import georegression.struct.homography.Homography2D_F64
-import kotlinx.coroutines.*
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
 import java.nio.file.Files
-import javax.imageio.ImageIO
 import kotlin.math.max
 import kotlin.math.pow
 import kotlin.random.Random
@@ -54,24 +57,30 @@ abstract class HomographyMapRunner(
         private const val minScroll = 75
 
         /**
-         * Difference theresholds
+         * Difference thresholds
          */
         private const val maxMapDiff = 80.0
         private const val maxSideDiff = 5.0
     }
 
+    private val metrics = Metrics()
+
+    private val predictor by lazy {
+        val model = MatchingModel(
+            config.assetsDirectory.resolve("models/SuperPoint.pt"),
+            config.assetsDirectory.resolve("models/SuperGlue.pt")
+        )
+        model.newPredictor(MatchingTranslator(480, 360)).also { it.setMetrics(metrics) }
+    }
+
     /**
      * Map homography cache
      */
     private var mapH: Homography2D_F64? = null
 
-    protected open val extractBlueNodes: Boolean = true
-    protected open val extractWhiteNodes: Boolean = false
-    protected open val extractYellowNodes: Boolean = true
-
     final override val nodes: List<MapNode>
 
-    val fullMap: GrayF32
+    val fullMap: Image
 
     init {
         val n = async(Dispatchers.IO) {
@@ -83,41 +92,41 @@ abstract class HomographyMapRunner(
             }
         }
         val fm = async(Dispatchers.IO) {
-            val path = config.assetsDirectory.resolve("$PREFIX/map.png")
-            if (Files.exists(path)) {
-                ImageIO.read(path.toFile()).extractNodes(extractBlueNodes, extractWhiteNodes, extractYellowNodes)
-            } else {
-                GrayF32()
-            }
+            ImageFactory.getInstance().fromFile(config.assetsDirectory.resolve("$PREFIX/map.png"))
         }
 
         nodes = runBlocking { n.await() }
         fullMap = runBlocking { fm.await() }
     }
 
+    override suspend fun cleanup() {
+        mapH = null
+    }
+
     override suspend fun MapNode.findRegion(): AndroidRegion {
         val window = mapRunnerRegions.window
-        var h: Homography2D_F64? = null
-        while (h == null) {
-            h = try {
-                mapH
-                    ?: fullMap.homography(window.capture().extractNodes(extractBlueNodes, extractWhiteNodes, extractYellowNodes))
-            } catch (e: IllegalStateException) {
-                continue
-            }
+
+        val h = mapH ?: run {
+            logger.info("Finding map transformation")
+            val prediction = predictor.predict(fullMap to ImageFactory.getInstance().fromImage(window.capture()))
+            logger.debug("Homography prediction metrics:")
+            logger.debug("Preprocess: ${metrics.latestMetric("Preprocess").value.toLong() / 1000000} ms")
+            logger.debug("Inference: ${metrics.latestMetric("Inference").value.toLong() / 1000000} ms")
+            logger.debug("Postprocess: ${metrics.latestMetric("Postprocess").value.toLong() / 1000000} ms")
+            logger.debug("Total: ${metrics.latestMetric("Total").value.toLong() / 1000000} ms")
+            mapH = prediction
+            prediction
         }
 
         suspend fun retry(): AndroidRegion {
-            if (Random.nextBoolean()) {
-                logger.info("Zoom out")
-                region.pinch(
-                    Random.nextInt(500, 700),
-                    Random.nextInt(300, 400),
-                    0.0,
-                    500
-                )
-                delay(1000)
-            }
+            logger.info("Zoom out")
+            region.pinch(
+                Random.nextInt(500, 700),
+                Random.nextInt(300, 400),
+                0.0,
+                500
+            )
+            delay(1000)
             mapH = null
             return findRegion()
         }
@@ -192,28 +201,6 @@ abstract class HomographyMapRunner(
             delay(200)
             return findRegion()
         }
-
-        while (isActive) {
-            val targets = mutableListOf<Pair<Int, Int>>()
-            val img = roi.capture().extractNodes()
-            for (y in 0 until img.height) {
-                var index = img.startIndex + y * img.stride
-                for (x in 0 until img.width) {
-                    if (img.data[index++] >= 175) targets += x to y
-                }
-            }
-            yield()
-            if (targets.isEmpty()) {
-                logger.debug("No targets found, retry")
-                if (Random.nextBoolean()) continue else return retry()
-            }
-            logger.debug("${targets.size} target candidates for node $this")
-            val target = targets.random().let { (cX, cY) ->
-                region.subRegionAs<AndroidRegion>(roi.x + cX, roi.y + cY, 5, 5)
-            }
-            logger.debug("Node target: (x=${target.x},y=${target.y})")
-            return target
-        }
-        throw NodeNotFoundException(this)
+        return roi
     }
 }

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/MapRunner.kt

@@ -116,9 +116,22 @@ abstract class MapRunner(
     abstract val isCorpseDraggingMap: Boolean
 
     /**
-     * Main execution function that is executed when map is entered
+     * Main run function that goes through whole life cycle of MapRunner
      */
-    abstract suspend fun execute()
+    suspend fun execute() {
+        begin()
+        cleanup()
+    }
+
+    /**
+     * Function that is executed when map is entered
+     */
+    abstract suspend fun begin()
+
+    /**
+     * Cleanup function run after execute()
+     */
+    open suspend fun cleanup() = Unit
 
     /**
      * Executes when entering a battle

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/maps/Map0_2.kt

@@ -25,7 +25,10 @@ import com.waicool20.wai2k.config.Wai2KProfile
 import com.waicool20.wai2k.script.ScriptRunner
 import com.waicool20.wai2k.script.modules.combat.HomographyMapRunner
 import com.waicool20.waicoolutils.logging.loggerFor
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.yield
+import kotlin.math.roundToLong
+import kotlin.random.Random
 
 class Map0_2(
     scriptRunner: ScriptRunner,
@@ -36,8 +39,17 @@ class Map0_2(
     private val logger = loggerFor<Map0_2>()
     override val isCorpseDraggingMap = true
 
-    override suspend fun execute() {
-        nodes[2].findRegion() // Try focus boss node to get map centered
+    override suspend fun begin() {
+        if (gameState.requiresMapInit) {
+            logger.info("Zoom out")
+            region.pinch(
+                Random.nextInt(900, 1000),
+                Random.nextInt(300, 400),
+                0.0,
+                1000)
+            delay((900 * gameState.delayCoefficient).roundToLong()) //Wait to settle
+            gameState.requiresMapInit = false
+        }
         val rEchelons = deployEchelons(nodes[14], nodes[13])
         mapRunnerRegions.startOperation.click(); yield()
         waitForGNKSplash()

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/maps/Map0_2_EX.kt

@@ -40,7 +40,7 @@ class Map0_2_EX(
     private val logger = loggerFor<Map0_2_EX>()
     override val isCorpseDraggingMap = true
 
-    override suspend fun execute() {
+    override suspend fun begin() {
 
         if (gameState.requiresMapInit) {
             // Check to see if its already good

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/maps/Map10_4E.kt

@@ -38,7 +38,7 @@ class Map10_4E(
     private val logger = loggerFor<Map10_4E>()
     override val isCorpseDraggingMap = false
 
-    override suspend fun execute() {
+    override suspend fun begin() {
         if (gameState.requiresMapInit) {
             logger.info("Zoom out")
             repeat(2) {

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/maps/Map10_4E_Drag.kt

@@ -43,7 +43,7 @@ class Map10_4E_Drag(
     private val logger = loggerFor<Map10_4E_Drag>()
     override val isCorpseDraggingMap = true
 
-    override suspend fun execute() {
+    override suspend fun begin() {
 
         // Mostly empty region to the left
         val r = region.subRegionAs<AndroidRegion>(300, 500, 150, 8)

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/maps/Map11_5.kt

@@ -38,7 +38,7 @@ class Map11_5(
     private val logger = loggerFor<Map11_5>()
     override val isCorpseDraggingMap = true
 
-    override suspend fun execute() {
+    override suspend fun begin() {
         // No need to zoom, delay for map lag
         delay((1000 * gameState.delayCoefficient).roundToLong())
         val rEchelons = deployEchelons(nodes[1], nodes[0])

src/main/kotlin/com/waicool20/wai2k/script/modules/combat/maps/Map1_6.kt

@@ -40,7 +40,7 @@ class Map1_6(
     private val logger = loggerFor<Map1_6>()
     override val isCorpseDraggingMap = false
 
-    override suspend fun execute() {
+    override suspend fun begin() {
         logger.info("Zoom out")
         region.pinch(
             Random.nextInt(700, 800),