From 9dcad4a9fbe1dee1534c78470461a514831bb999 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Wed, 11 Sep 2024 19:53:49 +0100 Subject: [PATCH] fxi java native libs fetching from releases page --- ...nsys.zkevm.linea-native-libs-helper.gradle | 95 ++++++------------- jvm-libs/blob-compressor/build.gradle | 29 +----- jvm-libs/blob-shnarf-calculator/build.gradle | 4 +- 3 files changed, 32 insertions(+), 96 deletions(-) diff --git a/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle b/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle index a4dccc53c..be2fc0333 100644 --- a/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle +++ b/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle @@ -1,59 +1,17 @@ -import groovy.json.JsonSlurper - import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.Path import java.time.Duration import java.time.Instant -static def downloadAndParseJson( - String url, - Map headers = [:] -) { - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection() - connection.requestMethod = 'GET' - connection.setRequestProperty('Accept', 'application/json') - headers.each { key, value -> - connection.setRequestProperty(key, value) - } - connection.connect() - if (connection.responseCode == HttpURLConnection.HTTP_OK) { - def jsonResponse = connection.inputStream.text - def jsonSlurper = new JsonSlurper() - return jsonSlurper.parseText(jsonResponse) - } else { - throw new GradleException("Failed load JSON from $url HTTP error code: ${connection.responseCode}") - } -} - - -static def getReleaseAsset( - String releaseTagName, - String githubAccessToken -) { - def urlStr = 'https://api.github.com/repos/ConsenSys/zkevm-monorepo/releases' - def json = downloadAndParseJson(urlStr, ['Authorization': "token ${githubAccessToken}"]) - def release = json.find { it.tag_name == releaseTagName } - if (release == null) { - def releases = json.collect { it.tag_name } - throw new GradleException("Release ${releaseTagName} not found! releases: ${releases}") - } - if (release.assets.size() == 0) { - throw new GradleException("Release ${releaseTagName} has no assets!") - } - def asset = release.assets.find { it.name.contains(releaseTagName) } - //println(JsonOutput.prettyPrint(JsonOutput.toJson(asset))) - asset -} - void downloadFileUsingWget( String url, - String githubAccessToken, String outputFilePath ) { - println("Downloading ${url.replace(githubAccessToken, "*****")} into ${outputFilePath}") - def command = "curl -L -H 'Accept:application/octet-stream' -u ${githubAccessToken}: -o ${outputFilePath} ${url}" -// println("# " + command) + println("Downloading ${url} into ${outputFilePath}") + + String command = "curl -L -H 'Accept:application/octet-stream' -o ${outputFilePath} ${url}" + // println("# " + command) def execResult = exec { commandLine 'bash', '-c', command @@ -72,12 +30,11 @@ ext.architectureResourceDirMapping = [ "linux_x86_64" : "linux-x86-64" ] -private String downloadReleaseAsset( +private String downloadAssetIfNotPresent( + String libsZipUrl, String releaseTag, - String outputDir, - String githubAccessToken + String outputDir ) { - def releaseAssetJsonInfo = getReleaseAsset(releaseTag.toString(), githubAccessToken) def fileName = releaseTag + ".zip" def outputFilePath = Path.of(outputDir).resolve(fileName) @@ -92,7 +49,7 @@ private String downloadReleaseAsset( } } - downloadFileUsingWget(releaseAssetJsonInfo.url.toString(), githubAccessToken, outputFilePath.toString()) + downloadFileUsingWget(libsZipUrl, outputFilePath.toString()) return outputFilePath.toString() } @@ -156,16 +113,12 @@ def lazyUnzipWithRetry( } } -def downloadReleaseAndExtractToResources( - String releaseTag, - String libName, - String outputDir, - String githubAccessToken +def extractLibToResources( + Path zipFile, + Path outputUnzipDir, + String libName ) { - def outputUnzipDir = Path.of(outputDir).resolve(releaseTag) - def outputFile = downloadReleaseAsset(releaseTag, outputDir, githubAccessToken) - lazyUnzipWithRetry(Path.of(outputFile), outputUnzipDir, Duration.ofSeconds(60)) - + lazyUnzipWithRetry(zipFile, outputUnzipDir, Duration.ofSeconds(60)) fileTree(outputUnzipDir.toFile()) .filter { it.name.contains(libName) && (it.name.endsWith(".so") || it.name.endsWith(".dylib")) } .each { File file -> @@ -190,14 +143,22 @@ def downloadReleaseAndExtractToResources( } } -ext.fetchLib = { +def downloadZipReleaseAndExtractToResources( + String libsZipUrl, String releaseTag, String libName, - String outputDir, - String githubAccessToken = System.getenv("GITHUB_TOKEN") + String outputDir +) { + def zipFile = downloadAssetIfNotPresent(libsZipUrl, releaseTag, outputDir) + def outputUnzipDir = Path.of(outputDir).resolve(releaseTag) + extractLibToResources(Path.of(zipFile), outputUnzipDir, libName) +} + +ext.fetchLibFromZip = { + String libsZipUrl, + String libName, + String outputDir -> - if (githubAccessToken == null) { - throw new GradleException("GITHUB_TOKEN is required") - } - downloadReleaseAndExtractToResources(releaseTag, libName, outputDir, githubAccessToken) + def releaseTag = libsZipUrl.split("/").last().replace(".zip", "") + downloadZipReleaseAndExtractToResources(libsZipUrl, releaseTag, libName, outputDir) } diff --git a/jvm-libs/blob-compressor/build.gradle b/jvm-libs/blob-compressor/build.gradle index 3ba407889..7e590c213 100644 --- a/jvm-libs/blob-compressor/build.gradle +++ b/jvm-libs/blob-compressor/build.gradle @@ -29,33 +29,8 @@ def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get() task downloadNativeLibs { doLast { - fetchLib("blob-libs-v0.1.0", "blob_compressor", libsZipDownloadOutputDir) - fetchLib("blob-libs-v1.0.1", "blob_compressor", libsZipDownloadOutputDir) - } -} - -// GITHUB_TOKEN=ghp_XXX ./gradlew jvm-libs:blob-compressor:downloadNativeLib -PreleaseTag=blob-libs-v0.1.0 -PlibName=blob_compressor -tasks.register('downloadNativeLib') { - def releaseTag - def libName - - doLast { - if (project.hasProperty("releaseTag")) { - releaseTag = project.releaseTag - } else { - throw new GradleException("releaseTag is required") - } - if (project.hasProperty("libName")) { - libName = project.libName - } else { - throw new GradleException("libName is required") - } - - fetchLib( - releaseTag, - libName, - libsZipDownloadOutputDir - ) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v0.1.0/linea-blob-libs-v0.1.0.zip", "blob_compressor", libsZipDownloadOutputDir) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.0.1/linea-blob-libs-v1.0.1.zip", "blob_compressor", libsZipDownloadOutputDir) } } diff --git a/jvm-libs/blob-shnarf-calculator/build.gradle b/jvm-libs/blob-shnarf-calculator/build.gradle index 0abcab5a8..6f337d50c 100644 --- a/jvm-libs/blob-shnarf-calculator/build.gradle +++ b/jvm-libs/blob-shnarf-calculator/build.gradle @@ -28,8 +28,8 @@ def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get() task downloadNativeLibs { doLast { - fetchLib("blob-libs-v0.1.0", "shnarf_calculator", libsZipDownloadOutputDir) - fetchLib("blob-libs-v1.0.1", "shnarf_calculator", libsZipDownloadOutputDir) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v0.1.0/linea-blob-libs-v0.1.0.zip", "shnarf_calculator", libsZipDownloadOutputDir) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.0.1/linea-blob-libs-v1.0.1.zip", "shnarf_calculator", libsZipDownloadOutputDir) } }