[tokenizers] Fixes huggingface build for Windows (#3330)

deepjavalibrary · Jul 11, 2024 · b171572 · b171572
1 parent 702d019
commit b171572
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/.github/workflows/native_s3_huggingface.yml b/.github/workflows/native_s3_huggingface.yml
@@ -115,7 +115,6 @@ jobs:
         shell: bash
         run: |
           DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
-          rm -f extensions/tokenizers/jnilib/$DJL_VERSION/win-x86_64/*.dll
           TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
           aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/win-x86_64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/win-x86_64/
           aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"

diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts
@@ -25,19 +25,23 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(),
-            "tokenizersVersion" to libs.versions.tokenizers.get(),
-            "version" to version))
+        inputs.properties(
+            mapOf(
+                "djlVersion" to libs.versions.djl.get(),
+                "tokenizersVersion" to libs.versions.tokenizers.get(),
+                "version" to version
+            )
+        )
         val baseResourcePath = "${project.projectDir}/build/resources/main"
         outputs.dirs(File("${baseResourcePath}/native/lib"), File("${baseResourcePath}/nlp"))
 
         doLast {
             var url = "https://publish.djl.ai/tokenizers"
             val (tokenizers, djl) = libs.versions.tokenizers.get() to libs.versions.djl.get()
             val files = mapOf(
-                "win-x86_64/libwinpthread-1.dll" to "extra",
-                "win-x86_64/libgcc_s_seh-1.dll" to "extra",
-                "win-x86_64/libstdc%2B%2B-6.dll" to "extra",
+                "win-x86_64/cpu/libwinpthread-1.dll" to "extra/win-x86_64/libwinpthread-1.dll",
+                "win-x86_64/cpu/libgcc_s_seh-1.dll" to "extra/win-x86_64/libgcc_s_seh-1.dll",
+                "win-x86_64/cpu/libstdc%2B%2B-6.dll" to "extra/win-x86_64/libstdc%2B%2B-6.dll",
                 "win-x86_64/cpu/tokenizers.dll" to "$tokenizers/jnilib/$djl",
                 "linux-x86_64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
                 "linux-aarch64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
@@ -49,7 +53,12 @@ tasks {
                 val file = jnilibDir / URLDecoder.decode(key, "UTF-8")
                 if (file.exists())
                     project.logger.lifecycle("prebuilt or cached file found for $key")
-                else if ("extra" == value || !project.hasProperty("jni")) {
+                else if (value.startsWith("extra")) {
+                    project.logger.lifecycle("Downloading $url/$value")
+                    file.parentFile.mkdirs()
+                    val downloadPath = "$url/$value".url
+                    downloadPath into file
+                } else if (!project.hasProperty("jni")) {
                     project.logger.lifecycle("Downloading $url/$value/$key")
                     file.parentFile.mkdirs()
                     val downloadPath = "$url/$value/$key".url