Merge branch 'main' into wrapped-sort-rexslot

substrait-io · Jul 27, 2024 · bc3fc8a · bc3fc8a
2 parents 00854d5 + 9b153e0
commit bc3fc8a
Show file tree

Hide file tree

Showing 129 changed files with 5,802 additions and 758 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -16,3 +16,9 @@ end_of_line = unset
 insert_final_newline = unset
 indent_style = unset
 trim_trailing_whitespace = unset
+
+[*.gradle.kts]
+indent_size = 2
+
+[.gitmodules]
+indent_style = tab
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -2,6 +2,11 @@ name: PR Build Check
 
 on:
   pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   editorconfig-checker:
     name: Check editorconfig
@@ -10,7 +15,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: editorconfig-checker/action-editorconfig-checker@main
+      - uses: editorconfig-checker/action-editorconfig-checker@v2
       - run: editorconfig-checker
   commitlint:
     name: Lint commits for semantic-release
@@ -28,7 +33,43 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gradle/wrapper-validation-action@v1
+      - uses: gradle/actions/wrapper-validation@v3
+  cyclonedx-sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Generate SBOMs
+        run: ./gradlew cyclonedxBom
+      - name: Upload SBOMs
+        uses: actions/upload-artifact@v4
+        with:
+          name: cyclonedx-sbom
+          path: |
+            core/build/reports/bom.json
+            isthmus/build/reports/bom.json
+            isthmus-cli/build/reports/bom.json
+  osv-scanner:
+    needs: cyclonedx-sbom
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - core
+          - isthmus
+          - isthmus-cli
+    steps:
+      - name: Download SBOMs
+        uses: actions/download-artifact@v4
+        with:
+          name: cyclonedx-sbom
+      - name: Scan
+        run: docker run --rm -v "${PWD}/${{ matrix.project }}/build/reports/bom.json:/bom.json" ghcr.io/google/osv-scanner --sbom /bom.json
   java:
     name: Build and Test Java
     runs-on: ubuntu-latest
@@ -37,10 +78,10 @@ jobs:
         with:
           submodules: recursive
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
-          distribution: 'adopt'
+          distribution: 'temurin'
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
       - name: Build with Gradle
@@ -71,17 +112,18 @@ jobs:
     - name: Build with Gradle
       run: gradle nativeImage
     - name: Smoke Test
-      run: ./isthmus/src/test/script/smoke.sh
-        ./isthmus/src/test/script/tpch_smoke.sh
+      run: |
+        ./isthmus-cli/src/test/script/smoke.sh
+        ./isthmus-cli/src/test/script/tpch_smoke.sh
     - name: Rename the artifact to OS-unique name
       shell: bash
       run: |
-        value=`mv isthmus/build/graal/isthmus isthmus/build/graal/isthmus-${{ matrix.os }}`
+        value=`mv isthmus-cli/build/graal/isthmus isthmus-cli/build/graal/isthmus-${{ matrix.os }}`
     - name: Publish artifact
       uses: actions/upload-artifact@v4
       with:
         name: isthmus-${{ matrix.os }}
-        path: isthmus/build/graal/isthmus-${{ matrix.os }}
+        path: isthmus-cli/build/graal/isthmus-${{ matrix.os }}
   dry-run-release:
     name: Dry-run release
     runs-on: ubuntu-latest

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -37,17 +37,18 @@ jobs:
       - name: Build with Gradle
         run: gradle nativeImage
       - name: Smoke Test
-        run: ./isthmus/src/test/script/smoke.sh
-          ./isthmus/src/test/script/tpch_smoke.sh
+        run: |
+          ./isthmus-cli/src/test/script/smoke.sh
+          ./isthmus-cli/src/test/script/tpch_smoke.sh
       - name: Rename the artifact to OS-unique name
         shell: bash
         run: |
-          value=`mv isthmus/build/graal/isthmus isthmus/build/graal/isthmus-${{ matrix.os }}`
+          value=`mv isthmus-cli/build/graal/isthmus isthmus-cli/build/graal/isthmus-${{ matrix.os }}`
       - name: Publish artifact
         uses: actions/upload-artifact@v4
         with:
           name: isthmus-${{ matrix.os }}
-          path: isthmus/build/graal/isthmus-${{ matrix.os }}
+          path: isthmus-cli/build/graal/isthmus-${{ matrix.os }}
   semantic-release:
     if: github.repository == 'substrait-io/substrait-java'
     runs-on: ubuntu-latest
@@ -58,10 +59,10 @@ jobs:
         with:
           submodules: recursive
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
-          distribution: 'adopt'
+          distribution: 'temurin'
       - uses: actions/setup-node@v4
         with:
           node-version: '20'

diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ gen
 *.iml
 out/**
 *.iws
+.vscode
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,130 @@
 Release Notes
 ---
 
+## [0.37.0](https://github.com/substrait-io/substrait-java/compare/v0.36.0...v0.37.0) (2024-07-21)
+
+### ⚠ BREAKING CHANGES
+
+* AdvancedExtension#getOptimization() has been removed. Use getOptimizations() instead.
+
+### Features
+
+* literal support for precision timestamp types ([#283](https://github.com/substrait-io/substrait-java/issues/283)) ([94996f9](https://github.com/substrait-io/substrait-java/commit/94996f916478ed8141e5fb54b1c8411cc80f4abd))
+* validate VirtualTableScan field names with schema ([#284](https://github.com/substrait-io/substrait-java/issues/284)) ([0f8514a](https://github.com/substrait-io/substrait-java/commit/0f8514a95f0ffa2c3cca645652ef91ebfd3ccb9d))
+
+### Miscellaneous Chores
+
+* update to substrait 0.52.0 ([#282](https://github.com/substrait-io/substrait-java/issues/282)) ([ada8d0b](https://github.com/substrait-io/substrait-java/commit/ada8d0be54b8bbd260b194c4e93f02ed42821b5d))
+
+## [0.36.0](https://github.com/substrait-io/substrait-java/compare/v0.35.0...v0.36.0) (2024-07-14)
+
+### ⚠ BREAKING CHANGES
+
+* Expression#options now returns List<FunctionOption>
+* ProtoAggregateFunctionConverter#from(AggregateFunction) now returns AggregateFunctionInvocation
+
+### Bug Fixes
+
+* include FunctionOptions when converting functions ([#278](https://github.com/substrait-io/substrait-java/issues/278)) ([e574913](https://github.com/substrait-io/substrait-java/commit/e57491333c7dd05ae3b1400e2185f807af1f5f88))
+
+## [0.35.0](https://github.com/substrait-io/substrait-java/compare/v0.34.0...v0.35.0) (2024-06-30)
+
+### Features
+
+* deprecate Timestamp and TimestampTZ visit functions ([#273](https://github.com/substrait-io/substrait-java/issues/273)) ([8a8253e](https://github.com/substrait-io/substrait-java/commit/8a8253ec1f077b81d0da6503e299662048fca825))
+* introduce substrait-spark module ([#271](https://github.com/substrait-io/substrait-java/issues/271)) ([8537dca](https://github.com/substrait-io/substrait-java/commit/8537dca93b410177f2ee5aefffe83f7c02a3668c))
+
+## [0.34.0](https://github.com/substrait-io/substrait-java/compare/v0.33.0...v0.34.0) (2024-06-23)
+
+### ⚠ BREAKING CHANGES
+
+* getDfsNames() has been removed from VirtualTableScan
+* getInitialSchema() not longer has a default implementation in VirtualTableScan
+
+### Bug Fixes
+
+* set VirtualTableScan schema explicitly ([#272](https://github.com/substrait-io/substrait-java/issues/272)) ([f1192cf](https://github.com/substrait-io/substrait-java/commit/f1192cfaf6c84fb1e466bae6eda75ba164444aa8))
+
+## [0.33.0](https://github.com/substrait-io/substrait-java/compare/v0.32.0...v0.33.0) (2024-06-16)
+
+### Features
+
+* **isthmus:** support for PrecisionTimestamp conversions ([#262](https://github.com/substrait-io/substrait-java/issues/262)) ([e726904](https://github.com/substrait-io/substrait-java/commit/e72690425cb31e52bc37550c1c4851db1b927651))
+
+### Bug Fixes
+
+* **isthmus:** correct SLF4J dependency ([#268](https://github.com/substrait-io/substrait-java/issues/268)) ([3134504](https://github.com/substrait-io/substrait-java/commit/31345045d522bf85bc60a59d52e4dd55601abbf8))
+
+## [0.32.0](https://github.com/substrait-io/substrait-java/compare/v0.31.0...v0.32.0) (2024-06-04)
+
+### ⚠ BREAKING CHANGES
+
+* Substrait FP32 is now mapped to Calcite REAL instead of FLOAT
+* Calcite FLOAT is now mapped to Substrait FP64 instead of FP32
+
+In Calcite, the Sql Type Names DOUBLE and FLOAT correspond to FP64, and REAL corresponds to FP32
+
+### Bug Fixes
+
+* account for struct fields in VirtualTableScan check ([#255](https://github.com/substrait-io/substrait-java/issues/255)) ([3bbcf82](https://github.com/substrait-io/substrait-java/commit/3bbcf82687bc51fdb1695436c198e91ba56befed))
+* map Calcite REAL to Substrait FP32 ([#261](https://github.com/substrait-io/substrait-java/issues/261)) ([37331c2](https://github.com/substrait-io/substrait-java/commit/37331c2fbee679fd5ec482d8ff4d16f1c7c1c5c0))
+
+## [0.31.0](https://github.com/substrait-io/substrait-java/compare/v0.30.0...v0.31.0) (2024-05-05)
+
+
+### ⚠ BREAKING CHANGES
+
+* **isthumus:** CLI related functionality is now in the io.substrait.isthmus.cli package
+
+### Features
+
+* allow deployment time selection of logging framework [#243](https://github.com/substrait-io/substrait-java/issues/243) ([#244](https://github.com/substrait-io/substrait-java/issues/244)) ([72bab63](https://github.com/substrait-io/substrait-java/commit/72bab63edf6c4ffb12c3c4b0e4f49d066e0c5524))
+* **isthumus:** extract CLI into isthmus-cli project [#248](https://github.com/substrait-io/substrait-java/issues/248) ([#249](https://github.com/substrait-io/substrait-java/issues/249)) ([a49de62](https://github.com/substrait-io/substrait-java/commit/a49de62c670274cccfa8b94fb86e88b36fc716d3))
+
+## [0.30.0](https://github.com/substrait-io/substrait-java/compare/v0.29.1...v0.30.0) (2024-04-28)
+
+
+### ⚠ BREAKING CHANGES
+
+* ParameterizedTypeVisitor has new visit methods
+* TypeExpressionVisitor has new visit methods
+* TypeVisitor has new visit methods
+* BaseProtoTypes has new visit methods
+
+### Bug Fixes
+
+* handle FetchRels with only offset set ([#252](https://github.com/substrait-io/substrait-java/issues/252)) ([b334e1d](https://github.com/substrait-io/substrait-java/commit/b334e1d4004ebc4598cab7bc076f3d477e97a52a))
+
+
+### Miscellaneous Chores
+
+* update to substrait 0.48.0 ([#250](https://github.com/substrait-io/substrait-java/issues/250)) ([572fe57](https://github.com/substrait-io/substrait-java/commit/572fe57ccf473e3d680f8928dd5f6833583350cc))
+
+## [0.29.1](https://github.com/substrait-io/substrait-java/compare/v0.29.0...v0.29.1) (2024-03-31)
+
+
+### Bug Fixes
+
+* correct function compound names for IntervalDay and IntervalYear [#240](https://github.com/substrait-io/substrait-java/issues/240) ([#242](https://github.com/substrait-io/substrait-java/issues/242)) ([856331b](https://github.com/substrait-io/substrait-java/commit/856331bae9901e618663622bbf60eaf923dea5b8))
+
+## [0.29.0](https://github.com/substrait-io/substrait-java/compare/v0.28.1...v0.29.0) (2024-03-17)
+
+
+### ⚠ BREAKING CHANGES
+
+* **isthmus:** method ExpressionCreator.cast(Type, Expression) has been removed
+
+### Features
+
+* **isthmus:** support for safe casting ([#236](https://github.com/substrait-io/substrait-java/issues/236)) ([72785ad](https://github.com/substrait-io/substrait-java/commit/72785ad1a4bd1ba8481d75ddaf4f1a822bf9bf6b))
+
+## [0.28.1](https://github.com/substrait-io/substrait-java/compare/v0.28.0...v0.28.1) (2024-03-10)
+
+
+### Bug Fixes
+
+* use coercive function matcher before least restrictive matcher ([#237](https://github.com/substrait-io/substrait-java/issues/237)) ([e7aa8ff](https://github.com/substrait-io/substrait-java/commit/e7aa8ff1fe11dd784074138bf75c1afa140b59db))
+
 ## [0.28.0](https://github.com/substrait-io/substrait-java/compare/v0.27.0...v0.28.0) (2024-02-25)
 
 

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -9,18 +9,23 @@ plugins {
   id("com.github.vlsi.gradle-extensions") version "1.74"
   id("com.diffplug.spotless") version "6.11.0"
   id("io.github.gradle-nexus.publish-plugin") version "1.1.0"
+  id("org.cyclonedx.bom") version "1.8.2"
 }
 
+var IMMUTABLES_VERSION = properties.get("immutables.version")
+var JUNIT_VERSION = properties.get("junit.version")
+var SLF4J_VERSION = properties.get("slf4j.version")
+
 repositories { mavenCentral() }
 
 java { toolchain { languageVersion.set(JavaLanguageVersion.of(17)) } }
 
 dependencies {
-  testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.2")
-  testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
-  implementation("org.slf4j:slf4j-jdk14:1.7.30")
-  annotationProcessor("org.immutables:value:2.8.8")
-  compileOnly("org.immutables:value-annotations:2.8.8")
+  testImplementation("org.junit.jupiter:junit-jupiter-api:${JUNIT_VERSION}")
+  testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${JUNIT_VERSION}")
+  implementation("org.slf4j:slf4j-api:${SLF4J_VERSION}")
+  annotationProcessor("org.immutables:value:${IMMUTABLES_VERSION}")
+  compileOnly("org.immutables:value-annotations:${IMMUTABLES_VERSION}")
   annotationProcessor("com.github.bsideup.jabel:jabel-javac-plugin:0.4.2")
   compileOnly("com.github.bsideup.jabel:jabel-javac-plugin:0.4.2")
 }
@@ -64,6 +69,21 @@ allprojects {
       }
     }
   }
+
+  if (listOf("core", "isthmus", "isthmus-cli").contains(project.name)) {
+    apply(plugin = "org.cyclonedx.bom")
+    tasks.cyclonedxBom {
+      setIncludeConfigs(listOf("runtimeClasspath"))
+      setSkipConfigs(listOf("compileClasspath", "testCompileClasspath"))
+      setProjectType("library")
+      setSchemaVersion("1.5")
+      setDestination(project.file("build/reports"))
+      setOutputName("bom")
+      setOutputFormat("json")
+      setIncludeBomSerialNumber(false)
+      setIncludeLicenseText(false)
+    }
+  }
 }
 
 nexusPublishing {

diff --git a/ci/release/dry_run.sh b/ci/release/dry_run.sh
@@ -29,7 +29,7 @@ npx --yes \
   -p "@semantic-release/changelog" \
   -p "@semantic-release/exec" \
   -p "@semantic-release/git" \
-  -p "conventional-changelog-conventionalcommits@6.1.0" \
+  -p "conventional-changelog-conventionalcommits" \
   semantic-release \
   --ci false \
   --dry-run \

diff --git a/ci/release/publish.sh b/ci/release/publish.sh
@@ -4,4 +4,4 @@
 set -euo pipefail
 
 gradle wrapper
-./gradlew clean :core:publishToSonatype :isthmus:publishToSonatype closeAndReleaseSonatypeStagingRepository
+./gradlew clean :core:publishToSonatype :isthmus:publishToSonatype :spark:publishToSonatype closeAndReleaseSonatypeStagingRepository
diff --git a/ci/release/run.sh b/ci/release/run.sh
@@ -11,6 +11,5 @@ npx --yes \
   -p "@semantic-release/github" \
   -p "@semantic-release/exec" \
   -p "@semantic-release/git" \
-  -p "conventional-changelog-conventionalcommits@6.1.0" \
+  -p "conventional-changelog-conventionalcommits" \
   semantic-release --ci
-
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ gen @@
     *.iml
     out/**
     *.iws
+    .vscode