From 172325de806d1fe5e62799406aa70f363683b1bc Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Thu, 22 Apr 2021 11:16:21 +0530 Subject: [PATCH 1/7] use jgit to filter modified files --- cli/build.gradle | 1 + cli/src/main/kotlin/norm/cli/NormCli.kt | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cli/build.gradle b/cli/build.gradle index d0c7bae..8edc7d7 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -11,5 +11,6 @@ mainClassName = "norm.cli.NormCliKt" dependencies { implementation project(":codegen") implementation "com.github.ajalt:clikt:2.7.1" + implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.11.0.202103091610-r' } diff --git a/cli/src/main/kotlin/norm/cli/NormCli.kt b/cli/src/main/kotlin/norm/cli/NormCli.kt index aa485ce..10a68df 100644 --- a/cli/src/main/kotlin/norm/cli/NormCli.kt +++ b/cli/src/main/kotlin/norm/cli/NormCli.kt @@ -10,6 +10,8 @@ import norm.api.NormApi import norm.fs.IO import norm.fs.globSearch import norm.util.withPgConnection +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.storage.file.FileRepositoryBuilder import java.io.File @@ -67,7 +69,9 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli // If dir is provided, relativize to itself inputDir?.let { dir -> - globSearch(dir, "**.sql").forEach { sqlFile -> + val fileList =globSearch(dir, "**.sql") + val modifiedFiles = fileList.filter { modifiedFilesFromGitOnly().contains(it) } + modifiedFiles.forEach { sqlFile -> IO(sqlFile, dir, outDir).process(normApi::generate) } } @@ -77,6 +81,14 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli IO(sqlFile, basePath, outDir).process(normApi::generate) } } + + fun modifiedFilesFromGitOnly(): List { + val builder = FileRepositoryBuilder() + val repo = builder.setGitDir(File("." + "/.git")).setMustExist(true) + .build() + val git = Git(repo) + return git.diff().call().map { diffEntry -> diffEntry.newPath } + } } From b7e4d77849b15d3e7dab9cb720bb0ee84ffe67ac Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Mon, 7 Jun 2021 13:15:57 +0530 Subject: [PATCH 2/7] If git repository is not found then generate code for all sql files --- cli/src/main/kotlin/norm/cli/NormCli.kt | 98 +++++++++++++++++-------- 1 file changed, 66 insertions(+), 32 deletions(-) diff --git a/cli/src/main/kotlin/norm/cli/NormCli.kt b/cli/src/main/kotlin/norm/cli/NormCli.kt index 10a68df..13b9600 100644 --- a/cli/src/main/kotlin/norm/cli/NormCli.kt +++ b/cli/src/main/kotlin/norm/cli/NormCli.kt @@ -27,41 +27,68 @@ fun main(args: Array) = NormCli().main(args) * Can use env variable to pass in sensitive information */ class NormCli : CliktCommand( // command name is inferred as norm-cli - name = "norm-codegen", - help = """ + name = "norm-codegen", + help = """ Generates Kotlin Source files for given SQL files using the Postgres database connection """ ) { - private val jdbcUrl by option("-j", "--jdbc-url", help = "JDBC connection URL (can use env var PG_JDBC_URL)", envvar = "PG_JDBC_URL") - .default("jdbc:postgresql://localhost/postgres") - - private val username by option("-u", "--username", help = "Username (can use env var PG_USERNAME)", envvar = "PG_USERNAME") - .default("postgres") - - private val password by option("-p", "--password", help = "Password (can use env var PG_PASSWORD)", envvar = "PG_PASSWORD") - .default("") - - private val basePath by option("-b", "--base-path", help = " relative path from this dir will be used to infer package name") - .file(canBeFile = false, canBeDir = true, mustExist = true) - .default(File(".")) // Current working dir - - private val inputFilesAsOpts by option("-f", "--file", help = "[Multiple] SQL files, the file path relative to base path (-b) will be used to infer package name") - .file(canBeFile = true, canBeDir = false, mustExist = true) - .multiple() - .unique() + private val jdbcUrl by option( + "-j", + "--jdbc-url", + help = "JDBC connection URL (can use env var PG_JDBC_URL)", + envvar = "PG_JDBC_URL" + ) + .default("jdbc:postgresql://localhost/postgres") + + private val username by option( + "-u", + "--username", + help = "Username (can use env var PG_USERNAME)", + envvar = "PG_USERNAME" + ) + .default("postgres") + + private val password by option( + "-p", + "--password", + help = "Password (can use env var PG_PASSWORD)", + envvar = "PG_PASSWORD" + ) + .default("") + + private val basePath by option( + "-b", + "--base-path", + help = " relative path from this dir will be used to infer package name" + ) + .file(canBeFile = false, canBeDir = true, mustExist = true) + .default(File(".")) // Current working dir + + private val inputFilesAsOpts by option( + "-f", + "--file", + help = "[Multiple] SQL files, the file path relative to base path (-b) will be used to infer package name" + ) + .file(canBeFile = true, canBeDir = false, mustExist = true) + .multiple() + .unique() private val sqlFiles by argument() // give meaningful name for CLI help message - .file(canBeFile = true, canBeDir = false, mustExist = true) - .multiple() - .unique() + .file(canBeFile = true, canBeDir = false, mustExist = true) + .multiple() + .unique() - private val inputDir by option("-d", "--in-dir", help = "Dir containing .sql files, relative path from this dir will be used to infer package name") - .file(canBeFile = false, canBeDir = true, mustExist = true) + private val inputDir by option( + "-d", + "--in-dir", + help = "Dir containing .sql files, relative path from this dir will be used to infer package name" + ) + .file(canBeFile = false, canBeDir = true, mustExist = true) private val outDir by option("-o", "--out-dir", help = "Output dir where source should be generated") - .file(canBeFile = false, canBeDir = true, mustExist = true) - .required() + .file(canBeFile = false, canBeDir = true, mustExist = true) + .required() override fun run() = withPgConnection(jdbcUrl, username, password) { connection -> @@ -69,8 +96,8 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli // If dir is provided, relativize to itself inputDir?.let { dir -> - val fileList =globSearch(dir, "**.sql") - val modifiedFiles = fileList.filter { modifiedFilesFromGitOnly().contains(it) } + val fileList = globSearch(dir, "**.sql") + val modifiedFiles = modifiedFilesFromGit(dir,fileList) modifiedFiles.forEach { sqlFile -> IO(sqlFile, dir, outDir).process(normApi::generate) } @@ -82,12 +109,19 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli } } - fun modifiedFilesFromGitOnly(): List { + fun modifiedFilesFromGit(directory: File, fileList: Sequence): Sequence { val builder = FileRepositoryBuilder() - val repo = builder.setGitDir(File("." + "/.git")).setMustExist(true) + val repo = builder.setGitDir(File(directory.parent + "/.git")).setMustExist(false) .build() - val git = Git(repo) - return git.diff().call().map { diffEntry -> diffEntry.newPath } + return when { + repo.objectDatabase.exists() -> { + val git = Git(repo) + val diff = git.diff().call().map { diffEntry -> diffEntry.newPath } + fileList.filter { file -> diff.contains(file) } + } + else -> fileList + } + } } From 9a005b6aa933d4eb9a9fa20719f7f9f09b099697 Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Mon, 7 Jun 2021 18:25:27 +0530 Subject: [PATCH 3/7] SK | generate code for untracked modified files in git if file is not added in index yet --- cli/src/main/kotlin/norm/cli/NormCli.kt | 11 +++++---- cli/src/test/kotlin/norm/cli/NormCliTest.kt | 23 +++++++++++++++++++ .../sql/employees/add-new-employee.sql | 2 ++ 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 cli/src/test/resources/gitrepository/sql/employees/add-new-employee.sql diff --git a/cli/src/main/kotlin/norm/cli/NormCli.kt b/cli/src/main/kotlin/norm/cli/NormCli.kt index 13b9600..ceee18e 100644 --- a/cli/src/main/kotlin/norm/cli/NormCli.kt +++ b/cli/src/main/kotlin/norm/cli/NormCli.kt @@ -109,17 +109,20 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli } } - fun modifiedFilesFromGit(directory: File, fileList: Sequence): Sequence { + fun modifiedFilesFromGit(directory: File, fileList: Sequence): List { val builder = FileRepositoryBuilder() val repo = builder.setGitDir(File(directory.parent + "/.git")).setMustExist(false) .build() return when { repo.objectDatabase.exists() -> { val git = Git(repo) - val diff = git.diff().call().map { diffEntry -> diffEntry.newPath } - fileList.filter { file -> diff.contains(file) } + return when { + git.status().call().untracked.isNotEmpty() -> git.status().call().untracked + git.status().call().modified.isNotEmpty() -> git.status().call().modified + else -> git.diff().call().map { diffEntry -> diffEntry.newPath } + }.map { File(directory.parent+"/"+it) }.filter { it.name.endsWith("sql") } } - else -> fileList + else -> fileList.toList() } } diff --git a/cli/src/test/kotlin/norm/cli/NormCliTest.kt b/cli/src/test/kotlin/norm/cli/NormCliTest.kt index e5c98a0..b5400bb 100644 --- a/cli/src/test/kotlin/norm/cli/NormCliTest.kt +++ b/cli/src/test/kotlin/norm/cli/NormCliTest.kt @@ -10,8 +10,16 @@ import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.kotest.matchers.string.contain import io.kotest.matchers.string.startWith +import io.mockk.Runs +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkConstructor import norm.test.utils.PgContainer import norm.test.utils.toArgs +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.ObjectDatabase +import org.eclipse.jgit.lib.Repository +import org.eclipse.jgit.storage.file.FileRepositoryBuilder import java.io.File class NormCliTest : StringSpec() { @@ -74,5 +82,20 @@ class NormCliTest : StringSpec() { exception.message should startWith("Invalid value for \"-f\":") exception.message should contain("is a directory") } + "Should generate kotlin file for untracked file in git"{ + + val git = mockk(relaxed = true) + mockkConstructor(Repository::class) + val mockObjectDB = mockk(relaxed = true) + every { git.status().call().untracked } returns emptySet() + every { git.status().call().modified } returns emptySet() + every { git.diff().call()} returns emptyList() + every { anyConstructed().objectDatabase } returns mockObjectDB + every { mockObjectDB.exists() } returns true + val args = toArgs("-d src/test/resources/gitrepository/sql -b src/test/resources/gitrepository/sql -o $outputDir ${pgStr()}") + NormCli().parse(args) + File("$outputDir/employees/AddNewEmployee.kt").exists() shouldBe true + + } } } diff --git a/cli/src/test/resources/gitrepository/sql/employees/add-new-employee.sql b/cli/src/test/resources/gitrepository/sql/employees/add-new-employee.sql new file mode 100644 index 0000000..053cfa6 --- /dev/null +++ b/cli/src/test/resources/gitrepository/sql/employees/add-new-employee.sql @@ -0,0 +1,2 @@ +INSERT INTO employees(first_name, last_name) +VALUES (:firstName, :lastName) From bdcdc47731f5958e149ddb279146b3b7bdcccfb5 Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Mon, 7 Jun 2021 18:59:55 +0530 Subject: [PATCH 4/7] MODIFY | Fix test for untracked file , add mock for repo builder --- cli/src/main/kotlin/norm/cli/NormCli.kt | 2 +- cli/src/test/kotlin/norm/cli/NormCliTest.kt | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cli/src/main/kotlin/norm/cli/NormCli.kt b/cli/src/main/kotlin/norm/cli/NormCli.kt index ceee18e..6eacc86 100644 --- a/cli/src/main/kotlin/norm/cli/NormCli.kt +++ b/cli/src/main/kotlin/norm/cli/NormCli.kt @@ -109,7 +109,7 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli } } - fun modifiedFilesFromGit(directory: File, fileList: Sequence): List { + private fun modifiedFilesFromGit(directory: File, fileList: Sequence): List { val builder = FileRepositoryBuilder() val repo = builder.setGitDir(File(directory.parent + "/.git")).setMustExist(false) .build() diff --git a/cli/src/test/kotlin/norm/cli/NormCliTest.kt b/cli/src/test/kotlin/norm/cli/NormCliTest.kt index b5400bb..c5c2a23 100644 --- a/cli/src/test/kotlin/norm/cli/NormCliTest.kt +++ b/cli/src/test/kotlin/norm/cli/NormCliTest.kt @@ -17,6 +17,7 @@ import io.mockk.mockkConstructor import norm.test.utils.PgContainer import norm.test.utils.toArgs import org.eclipse.jgit.api.Git +import org.eclipse.jgit.internal.storage.file.FileRepository import org.eclipse.jgit.lib.ObjectDatabase import org.eclipse.jgit.lib.Repository import org.eclipse.jgit.storage.file.FileRepositoryBuilder @@ -85,13 +86,15 @@ class NormCliTest : StringSpec() { "Should generate kotlin file for untracked file in git"{ val git = mockk(relaxed = true) - mockkConstructor(Repository::class) - val mockObjectDB = mockk(relaxed = true) - every { git.status().call().untracked } returns emptySet() - every { git.status().call().modified } returns emptySet() - every { git.diff().call()} returns emptyList() - every { anyConstructed().objectDatabase } returns mockObjectDB - every { mockObjectDB.exists() } returns true + val repo = mockk(relaxed = true) + mockkConstructor(FileRepositoryBuilder::class) + mockkConstructor(Git::class) + every { anyConstructed().setGitDir(any()).setMustExist(any()).build() } returns repo + every { repo.objectDatabase.exists() } returns true + every { anyConstructed().status().call().untracked } returns setOf("sql/employees/add-new-employee.sql") + every { anyConstructed().status().call().modified } returns emptySet() + every { anyConstructed().diff().call() } returns emptyList() + val args = toArgs("-d src/test/resources/gitrepository/sql -b src/test/resources/gitrepository/sql -o $outputDir ${pgStr()}") NormCli().parse(args) File("$outputDir/employees/AddNewEmployee.kt").exists() shouldBe true From 5a69ff48b96eec521ad6c34aa9b1c0d5681a0361 Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Mon, 7 Jun 2021 19:12:54 +0530 Subject: [PATCH 5/7] MODIFY | clean up tests , add mockk dependency for cli tests --- cli/build.gradle | 1 + cli/src/test/kotlin/norm/cli/NormCliTest.kt | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cli/build.gradle b/cli/build.gradle index 8edc7d7..ff2bbd1 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -11,6 +11,7 @@ mainClassName = "norm.cli.NormCliKt" dependencies { implementation project(":codegen") implementation "com.github.ajalt:clikt:2.7.1" + testImplementation "io.mockk:mockk:1.11.0" implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.11.0.202103091610-r' } diff --git a/cli/src/test/kotlin/norm/cli/NormCliTest.kt b/cli/src/test/kotlin/norm/cli/NormCliTest.kt index c5c2a23..3f1daa0 100644 --- a/cli/src/test/kotlin/norm/cli/NormCliTest.kt +++ b/cli/src/test/kotlin/norm/cli/NormCliTest.kt @@ -10,7 +10,6 @@ import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.kotest.matchers.string.contain import io.kotest.matchers.string.startWith -import io.mockk.Runs import io.mockk.every import io.mockk.mockk import io.mockk.mockkConstructor @@ -18,8 +17,6 @@ import norm.test.utils.PgContainer import norm.test.utils.toArgs import org.eclipse.jgit.api.Git import org.eclipse.jgit.internal.storage.file.FileRepository -import org.eclipse.jgit.lib.ObjectDatabase -import org.eclipse.jgit.lib.Repository import org.eclipse.jgit.storage.file.FileRepositoryBuilder import java.io.File @@ -85,7 +82,6 @@ class NormCliTest : StringSpec() { } "Should generate kotlin file for untracked file in git"{ - val git = mockk(relaxed = true) val repo = mockk(relaxed = true) mockkConstructor(FileRepositoryBuilder::class) mockkConstructor(Git::class) From b0eadc0261671ab6b75509e8a6f6fb358c9267b8 Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Mon, 7 Jun 2021 19:38:31 +0530 Subject: [PATCH 6/7] MODIFY | remove unnecessary vals --- cli/src/main/kotlin/norm/cli/NormCli.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/src/main/kotlin/norm/cli/NormCli.kt b/cli/src/main/kotlin/norm/cli/NormCli.kt index 6eacc86..fca1c91 100644 --- a/cli/src/main/kotlin/norm/cli/NormCli.kt +++ b/cli/src/main/kotlin/norm/cli/NormCli.kt @@ -110,12 +110,14 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli } private fun modifiedFilesFromGit(directory: File, fileList: Sequence): List { - val builder = FileRepositoryBuilder() - val repo = builder.setGitDir(File(directory.parent + "/.git")).setMustExist(false) + + val repo = FileRepositoryBuilder() + .setGitDir(File(directory.parent + "/.git")) + .setMustExist(false) .build() + val git = Git(repo) return when { repo.objectDatabase.exists() -> { - val git = Git(repo) return when { git.status().call().untracked.isNotEmpty() -> git.status().call().untracked git.status().call().modified.isNotEmpty() -> git.status().call().modified @@ -124,7 +126,6 @@ class NormCli : CliktCommand( // command name is inferred as norm-cli } else -> fileList.toList() } - } } From 37062d921b38e35eecd6ad32f7c5199d717aa2d8 Mon Sep 17 00:00:00 2001 From: Swanand Keskar Date: Mon, 6 Sep 2021 17:14:37 +0530 Subject: [PATCH 7/7] remove duplicate clikt from build gradle --- cli/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/build.gradle b/cli/build.gradle index 0c2ce1c..5a4d35b 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -11,7 +11,6 @@ mainClassName = "norm.cli.NormCliKt" dependencies { implementation project(":codegen") implementation "com.github.ajalt:clikt:2.8.0" - implementation "com.github.ajalt:clikt:2.7.1" testImplementation "io.mockk:mockk:1.11.0" implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.11.0.202103091610-r' }