Improved incremental build (#3231)

* Update build files to allow better incremental and cached builds * Fix Spotbugs having conflicting paths for different tasks * Fixes build script for properties generation --------- Co-authored-by: Frank Liu <[email protected]>
deepjavalibrary · Jun 2, 2024 · ee9fd35 · ee9fd35
1 parent ab1490e
commit ee9fd35
Show file tree

Hide file tree

Showing 32 changed files with 196 additions and 85 deletions.
diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml
@@ -85,10 +85,9 @@ jobs:
         run: ./gradlew :engines:pytorch:pytorch-native:compileJNI
       - name: Test ONNX Runtime
         run: |
-          ./gradlew -Dai.djl.default_engine=OnnxRuntime :integration:test --stacktrace
-          ./gradlew :integration:clean
+          ./gradlew -Dai.djl.default_engine=OnnxRuntime :integration:clean :integration:test --stacktrace
       - name: Build with Gradle
-        run: ./gradlew build :jacoco:testCodeCoverageReport --stacktrace
+        run: ./gradlew clean build :jacoco:testCodeCoverageReport --stacktrace
       - name: Upload test results
         uses: actions/upload-artifact@v3
         if: always()

diff --git a/api/build.gradle.kts b/api/build.gradle.kts
@@ -23,12 +23,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.file(buildDirectory / "classes/java/main/ai/djl/engine/api.properties")
-        doFirst {
-            val classesDir = file("$buildDirectory/classes/java/main/ai/djl/engine/")
-            classesDir.mkdirs()
-            val propFile = File(classesDir, "api.properties")
-            propFile.text = "djl_version=${project.version}"
+        inputs.properties(mapOf("version" to version))
+        filesMatching("**/api.properties") {
+            expand(mapOf("version" to version))
         }
     }
 

diff --git a/api/src/main/resources/ai/djl/engine/api.properties b/api/src/main/resources/ai/djl/engine/api.properties
@@ -0,0 +1 @@
+djl_version=${version}
diff --git a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
@@ -15,6 +15,7 @@ spotbugs {
     excludeFilter = file("${rootProject.projectDir}/tools/conf/findbugs-exclude.xml")
     ignoreFailures = false
 }
+
 tasks {
     named<SpotBugsTask>("spotbugsMain") {
         reports.create("html") {
@@ -26,7 +27,7 @@ tasks {
         enabled = true
         reports.create("html") {
             required = true
-            outputLocation = file("$buildDirectory/reports/spotbugs.html")
+            outputLocation = file("$buildDirectory/reports/test/spotbugs.html")
         }
     }
 }
@@ -37,6 +38,7 @@ pmd {
     ruleSets = emptyList() // workaround pmd gradle plugin bug
     ruleSetFiles = files("${rootProject.projectDir}/tools/conf/pmd.xml")
 }
+
 tasks.withType<Pmd> {
     reports {
         xml.required = true
@@ -54,6 +56,7 @@ checkstyle {
     )
     configFile = file("${rootProject.projectDir}/tools/conf/checkstyle.xml")
 }
+
 tasks {
     named<Checkstyle>("checkstyleMain") {
         classpath += configurations["compileClasspath"]

diff --git a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts
@@ -20,6 +20,10 @@ tasks {
     }
 
     val verifyJava by registering {
+        val resultFilePath = "build/verifyJava-result.txt"
+        inputs.files(project.sourceSets.flatMap { it.allSource })
+        inputs.files(project.fileTree("generated-src"))
+        outputs.file(project.file(resultFilePath))
         doLast {
             val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`)
             for (item in project.sourceSets)
@@ -35,6 +39,7 @@ tasks {
                                     + "See https://github.com/deepjavalibrary/djl/blob/master/docs/development/development_guideline.md#coding-conventions for more details"
                         )
                 }
+            project.file(resultFilePath).writeText("Success")
         }
     }
 

diff --git a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
@@ -49,15 +49,18 @@ tasks {
         }
 
         jvmArgs("--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED")
-        systemProperties = System.getProperties().toMap() as Map<String, Any>
-        systemProperties.remove("user.dir")
-        // systemProperty "ai.djl.logging.level", "debug"
+        for (prop in System.getProperties().iterator()) {
+            val key = prop.key.toString()
+            if (key.startsWith("ai.djl.")) {
+                systemProperty(key, prop.value)
+            }
+        }
         systemProperties(
             "org.slf4j.simpleLogger.defaultLogLevel" to "debug",
             "org.slf4j.simpleLogger.log.org.mortbay.log" to "warn",
             "org.slf4j.simpleLogger.log.org.testng" to "info",
             "disableProgressBar" to "true",
-            "nightly" to System.getProperty("nightly", "false")
+            "nightly" to System.getProperty("nightly", "false"),
         )
         if (gradle.startParameter.isOffline)
             systemProperty("ai.djl.offline", "true")

diff --git a/engines/llama/build.gradle.kts b/engines/llama/build.gradle.kts
@@ -17,7 +17,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(project.projectDir / "build/classes/java/main/native/lib")
+        val path = "${project.projectDir}/build/resources/main"
+        inputs.properties(mapOf("djl_version" to libs.versions.djl.get(), "llamacpp_version" to libs.versions.llamacpp.get()))
+        outputs.dir("$path/native/lib")
         doLast {
             val llamacpp = libs.versions.llamacpp.get()
             val djl = libs.versions.djl.get()
@@ -49,15 +51,15 @@ tasks {
             }
             copy {
                 from(jnilibDir)
-                into(project.projectDir / "build/classes/java/main/native/lib")
+                into("$path/native/lib")
             }
 
             // write properties
-            val propFile = project.projectDir / "build/classes/java/main/native/lib/llama.properties"
+            val propFile = file("$path/native/lib/llama.properties")
             propFile.text = "version=$llamacpp-$version\n"
 
             url = "https://mlrepo.djl.ai/model/nlp/text_generation/ai/djl/huggingface/gguf/models.json.gz"
-            val prefix = project.projectDir / "build/classes/java/main/nlp/text_generation"
+            val prefix = File("$path/nlp/text_generation")
             val file = prefix / "ai.djl.huggingface.gguf.json"
             if (file.exists())
                 project.logger.lifecycle("gguf index file already exists")

diff --git a/engines/ml/xgboost/build.gradle.kts b/engines/ml/xgboost/build.gradle.kts
@@ -43,7 +43,8 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        val jnilibDir = buildDirectory / "classes/java/main/lib/linux/aarch64/"
+        val jnilibDir = buildDirectory / "resources/java/main/lib/linux/aarch64/"
+        inputs.properties(mapOf("xgboostVersion" to libs.versions.xgboost.get()))
         outputs.dir(jnilibDir)
         doLast {
             val url =

diff --git a/engines/mxnet/jnarator/build.gradle.kts b/engines/mxnet/jnarator/build.gradle.kts
@@ -20,6 +20,7 @@ tasks {
     pmdMain { exclude("ai/djl/mxnet/jnarator/parser/*") }
 
     jar {
+        dependsOn(generateGrammarSource)
         manifest {
             attributes(
                 "Main-Class" to "ai.djl.mxnet.jnarator.Main",
@@ -30,4 +31,13 @@ tasks {
         duplicatesStrategy = DuplicatesStrategy.INCLUDE
         from(configurations.runtimeClasspath.get().map { if (it.isDirectory()) it else zipTree(it) })
     }
+
+    generateGrammarSource {
+        dependsOn(verifyJava)
+    }
+
+    generateTestGrammarSource {
+        dependsOn(verifyJava)
+    }
+
 }
diff --git a/engines/mxnet/mxnet-engine/build.gradle.kts b/engines/mxnet/mxnet-engine/build.gradle.kts
@@ -25,11 +25,9 @@ sourceSets.main {
 
 tasks {
     processResources {
-        doFirst {
-            val classesDir = buildDirectory / "classes/java/main/"
-            classesDir.mkdirs()
-            val file = classesDir / "mxnet-engine.properties"
-            file.text = "djl_version=${libs.versions.djl.get()}\nmxnet_version=" + libs.versions.mxnet.get()
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get()))
+        filesMatching("**/mxnet-engine.properties") {
+            expand(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get()))
         }
     }
 
@@ -39,7 +37,11 @@ tasks {
     val jnarator by registering {
         val jnaratorJar = project(":engines:mxnet:jnarator").tasks.jar
         dependsOn(jnaratorJar)
+        inputs.file("${project.projectDir}/src/main/jna/mapping.properties")
+            .withPathSensitivity(PathSensitivity.RELATIVE)
+            .withPropertyName("jna/mapping.properties file")
         outputs.dir(buildDirectory / "generated-src")
+        outputs.cacheIf { true }
         doLast {
             val jnaGenerator = jnaratorJar.get().outputs.files.singleFile
             javaexec {
@@ -106,6 +108,7 @@ tasks {
     }
     compileJava { dependsOn(jnarator) }
     javadoc { dependsOn(jnarator) }
+    verifyJava { dependsOn(jnarator) }
 
     publishing {
         publications {

diff --git a/engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties b/engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties
@@ -0,0 +1,2 @@
+djl_version=${djlVersion}
+mxnet_version=${mxnetVersion}
diff --git a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts
@@ -20,7 +20,8 @@ dependencies {
 
 tasks {
     processResources {
-        outputs.dir(projectDir / "build/classes/java/main/nlp")
+        var basePath = "${project.projectDir}/build/resources/main/nlp"
+        outputs.dir(basePath)
         doLast {
             val url = "https://mlrepo.djl.ai/model/nlp"
             val tasks = listOf(
@@ -30,9 +31,8 @@ tasks {
                 "text_embedding",
                 "token_classification"
             )
-            val prefix = projectDir / "build/classes/java/main/nlp"
             for (task in tasks) {
-                val file = prefix / task / "ai.djl.huggingface.onnxruntime.json"
+                val file = File("$basePath/$task/ai.djl.huggingface.onnxruntime.json")
                 if (file.exists())
                     project.logger.lifecycle("model zoo metadata alrady exists: $task")
                 else {

diff --git a/engines/pytorch/pytorch-engine/build.gradle.kts b/engines/pytorch/pytorch-engine/build.gradle.kts
@@ -21,12 +21,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.file(buildDirectory / "classes/java/main/pytorch-engine.properties")
-        doFirst {
-            val classesDir = buildDirectory / "classes/java/main/"
-            classesDir.mkdirs()
-            val propFile = classesDir / "pytorch-engine.properties"
-            propFile.text = "djl_version=${libs.versions.djl.get()}\npytorch_version=${libs.versions.pytorch.get()}"
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get()))
+        filesMatching("**/pytorch-engine.properties") {
+            expand(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get()))
         }
     }
 

diff --git a/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties b/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties
@@ -0,0 +1,2 @@
+djl_version=${djlVersion}
+pytorch_version=${pytorchVersion}
diff --git a/engines/tensorflow/tensorflow-engine/build.gradle.kts b/engines/tensorflow/tensorflow-engine/build.gradle.kts
@@ -14,11 +14,9 @@ dependencies {
 }
 
 tasks.processResources {
-    doFirst {
-        val classesDir = buildDirectory / "classes/java/main/"
-        classesDir.mkdirs()
-        val file = classesDir / "tensorflow-engine.properties"
-        file.text = "djl_version=${libs.versions.djl.get()}\ntensorflow_version=${libs.versions.tensorflow.get()}"
+    inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "tensorflowVersion" to libs.versions.tensorflow.get()))
+    filesMatching("**/tensorflow-engine.properties") {
+        expand(mapOf("djlVersion" to libs.versions.djl.get(), "tensorflowVersion" to libs.versions.tensorflow.get()))
     }
 }
 

diff --git a/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
@@ -0,0 +1,2 @@
+djl_version=${djlVersion}
+tensorflow_version=${tensorflowVersion}
diff --git a/engines/tensorrt/build.gradle.kts b/engines/tensorrt/build.gradle.kts
@@ -21,7 +21,10 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(buildDirectory / "classes/java/main/native/lib")
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl, "trtVersion" to libs.versions.tensorrt.get(),
+            "version" to version))
+        val baseResourcePath = "${project.projectDir}/build/resources/main"
+        outputs.dir(file("${baseResourcePath}/native/lib"))
         doLast {
             val trtVersion = libs.versions.tensorrt.get()
             val djlVersion = libs.versions.djl.get()
@@ -41,12 +44,12 @@ tasks {
 
             copy {
                 from(jnilibDir)
-                into(buildDirectory / "classes/java/main/native/lib")
+                into("$baseResourcePath/native/lib")
             }
+        }
 
-            // write properties
-            val propFile = buildDirectory / "classes/java/main/native/lib/tensorrt.properties"
-            propFile.text = "version=${trtVersion}-${version}\n"
+        filesMatching("**/tensorrt.properties") {
+            expand(mapOf("trtVersion" to libs.versions.tensorrt.get(), "version" to version))
         }
     }
 

diff --git a/engines/tensorrt/src/main/resources/tensorrt.properties b/engines/tensorrt/src/main/resources/tensorrt.properties
@@ -0,0 +1 @@
+version=${trtVersion}-${version}
diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
@@ -1,5 +1,3 @@
-@file:Suppress("UNCHECKED_CAST")
-
 plugins {
     ai.djl.javaProject
     application
@@ -32,22 +30,28 @@ tasks {
 
     run.configure {
         environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out
-        // @Niels Doucet
-        // Just a heads-up: gradle support warned me about systemProperties System.getProperties(). It's really
-        // dangerous to just copy over all system properties to a task invocation. You should really be specific about
-        // the properties you'd like to expose inside the task, or you might get very strange issues.
-        systemProperties = System.getProperties().toMap() as Map<String, Any>
-        systemProperties.remove("user.dir")
-        systemProperty("file.encoding", "UTF-8")
+        for (prop in System.getProperties().iterator()) {
+            val key = prop.key.toString()
+            if (key.startsWith("ai.djl.")) {
+                systemProperty(key, prop.value)
+            }
+        }
     }
 
     register<JavaExec>("listmodels") {
-        systemProperties(System.getProperties() as Map<String, Any>)
-        systemProperties.remove("user.dir")
-        systemProperty("file.encoding", "UTF-8")
+        for (prop in System.getProperties().iterator()) {
+            val key = prop.key.toString()
+            if (key.startsWith("ai.djl.")) {
+                systemProperty(key, prop.value)
+            }
+        }
+        if (!systemProperties.containsKey("ai.djl.logging.level")) {
+            systemProperty("ai.djl.logging.level", "debug")
+        }
         classpath = sourceSets.main.get().runtimeClasspath
         mainClass = "ai.djl.examples.inference.ListModels"
     }
+
     distTar { enabled = false }
     distZip { enabled = false }
 }
diff --git a/extensions/fasttext/build.gradle.kts b/extensions/fasttext/build.gradle.kts
@@ -18,7 +18,11 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(buildDirectory / "classes/java/main/native/lib")
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(),
+            "fasttextVersion" to libs.versions.fasttext.get(),
+            "version" to version))
+        val baseResourcePath = "${project.projectDir}/build/resources/main"
+        outputs.dir("$baseResourcePath/native/lib")
         doLast {
             val url =
                 "https://publish.djl.ai/fasttext-${libs.versions.fasttext.get()}/jnilib/${libs.versions.djl.get()}"
@@ -41,12 +45,12 @@ tasks {
             }
             copy {
                 from(jnilibDir)
-                into(buildDirectory / "classes/java/main/native/lib")
+                into("$baseResourcePath/native/lib")
             }
+        }
 
-            // write properties
-            val propFile = buildDirectory / "classes/java/main/native/lib/fasttext.properties"
-            propFile.text = "version=${libs.versions.fasttext.get()}-${version}\n"
+        filesMatching("**/fasttext.properties") {
+            expand(mapOf("fasttextVersion" to libs.versions.fasttext.get(), "version" to version))
         }
     }
 

diff --git a/extensions/fasttext/src/main/resources/native/lib/fasttext.properties b/extensions/fasttext/src/main/resources/native/lib/fasttext.properties
@@ -0,0 +1 @@
+version=${fasttextVersion}-${version}