Skip to content

Commit

Permalink
Merge branch 'main' into examples-doc
Browse files Browse the repository at this point in the history
  • Loading branch information
mbwhite authored Dec 3, 2024
2 parents daf3fd7 + 55683fb commit 19d8c55
Show file tree
Hide file tree
Showing 143 changed files with 27,340 additions and 23 deletions.
1 change: 1 addition & 0 deletions .flake8
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
ignore = E203, E266, E501, W503, F403, F401
max-line-length = 88
select = B,C,E,F,W,T4,B9
exclude = tests/coverage/antlr_parser/*.py
6 changes: 4 additions & 2 deletions .github/workflows/licence_check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,7 @@ jobs:

- name: Check License Header
uses: enarx/spdx@master
with:
licenses: Apache-2.0 MIT
with:
licenses: |-
Apache-2.0
MIT
8 changes: 4 additions & 4 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@v1.44.0
- uses: bufbuild/buf-setup-action@v1.47.2
with:
github_token: ${{ github.token }}
- run: buf format --diff --exit-code
Expand All @@ -36,7 +36,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@v1.44.0
- uses: bufbuild/buf-setup-action@v1.47.2
with:
github_token: ${{ github.token }}
- uses: bufbuild/buf-lint-action@v1
Expand Down Expand Up @@ -72,7 +72,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: bufbuild/buf-setup-action@v1.44.0
- uses: bufbuild/buf-setup-action@v1.47.2
- uses: actions/setup-node@v4
with:
node-version: "20"
Expand All @@ -93,7 +93,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@v1.44.0
- uses: bufbuild/buf-setup-action@v1.47.2
- name: Run proto-prefix.py
run: tools/proto_prefix.py output test proto go_package=github.com/test/proto
- name: Modify buf config to build rewritten proto files
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr_breaking.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@v1.44.0
- uses: bufbuild/buf-setup-action@v1.47.2
with:
github_token: ${{ github.token }}
- name: check for breaking changes
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
with:
node-version: "20"

- uses: bufbuild/buf-setup-action@v1.44.0
- uses: bufbuild/buf-setup-action@v1.47.2
with:
github_token: ${{ github.token }}

Expand Down
7 changes: 7 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,10 @@ repos:
rev: 6.1.0
hooks:
- id: flake8
- repo: local
hooks:
- id: check-substrait-extensions_coverage
name: Check Substrait extensions and test coverage
entry: pytest tests/test_extensions.py::test_substrait_extension_coverage
language: python
pass_filenames: false
58 changes: 58 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,64 @@
Release Notes
---

## [0.62.0](https://github.com/substrait-io/substrait/compare/v0.61.0...v0.62.0) (2024-11-24)

### Features

* add readme for testcase file format ([#746](https://github.com/substrait-io/substrait/issues/746)) ([708a7b8](https://github.com/substrait-io/substrait/commit/708a7b811641c3384f826f69b9e8247d973c49b9))
* port function testcases from bft ([#738](https://github.com/substrait-io/substrait/issues/738)) ([d84ccd1](https://github.com/substrait-io/substrait/commit/d84ccd1262019fc05dbb32646cfbd71f9800d78a))

### Bug Fixes

* fix function lookup in coverage tool ([#744](https://github.com/substrait-io/substrait/issues/744)) ([3d2ff77](https://github.com/substrait-io/substrait/commit/3d2ff77575a7177f82a4d5b53408a059e9818922))

## [0.61.0](https://github.com/substrait-io/substrait/compare/v0.60.0...v0.61.0) (2024-11-17)

### Features

* add substrait test files to go embedded fs ([#740](https://github.com/substrait-io/substrait/issues/740)) ([e3a7773](https://github.com/substrait-io/substrait/commit/e3a7773895f4121deb6904b79efbcd76c656c9e8))
* handle parsing of list arguments in func testcases ([#737](https://github.com/substrait-io/substrait/issues/737)) ([1f9c710](https://github.com/substrait-io/substrait/commit/1f9c710cb45203bbabaaccd83b792b0b6739ae3a))
* update operator to update a table ([#734](https://github.com/substrait-io/substrait/issues/734)) ([adb1079](https://github.com/substrait-io/substrait/commit/adb1079ab60480d3b7293840bcb923a0f0612211))

### Bug Fixes

* misc fixes and cleanup to func testcase grammar ([#742](https://github.com/substrait-io/substrait/issues/742)) ([ea994c2](https://github.com/substrait-io/substrait/commit/ea994c2ead91eececb770cca1e7f0268e0ebb2ad))

## [0.60.0](https://github.com/substrait-io/substrait/compare/v0.59.0...v0.60.0) (2024-11-10)

### Features

* add antlr grammar for test file format ([#728](https://github.com/substrait-io/substrait/issues/728)) ([752aa63](https://github.com/substrait-io/substrait/commit/752aa636c01647ef15fd59edfd675533e881e692))
* add CreateMode for CTAS in WriteRel ([#715](https://github.com/substrait-io/substrait/issues/715)) ([2e13d0b](https://github.com/substrait-io/substrait/commit/2e13d0b7b4a2b5ac8221e7f02cc8f04456c8f11c))
* update test file format to support aggregate functions ([#736](https://github.com/substrait-io/substrait/issues/736)) ([c18c0c1](https://github.com/substrait-io/substrait/commit/c18c0c1347376efa4dfab503bb4db9f820df3cf3))

### Bug Fixes

* typo in site/docs/tutorial/sql_to_substrait.md ([#735](https://github.com/substrait-io/substrait/issues/735)) ([9cccb04](https://github.com/substrait-io/substrait/commit/9cccb04fba336489b70ed42b71f73a0a1e34f9f5))

## [0.59.0](https://github.com/substrait-io/substrait/compare/v0.58.0...v0.59.0) (2024-11-03)

### ⚠ BREAKING CHANGES

* changes the message type for Expressions field in
VirtualTable

### Features

* add antlr grammar for types ([#730](https://github.com/substrait-io/substrait/issues/730)) ([820085f](https://github.com/substrait-io/substrait/commit/820085fc913692147d0c8fdfcbf289fb8b348835))

### Bug Fixes

* virtualTable expression should represent a row of expression ([#727](https://github.com/substrait-io/substrait/issues/727)) ([a2df42c](https://github.com/substrait-io/substrait/commit/a2df42c76282854d8674aa664d69dacd43630551))

## [0.58.0](https://github.com/substrait-io/substrait/compare/v0.57.1...v0.58.0) (2024-10-13)

### Features

* define sideband optimization hints ([#705](https://github.com/substrait-io/substrait/issues/705)) ([e386a29](https://github.com/substrait-io/substrait/commit/e386a29377c4138a6a2aee87750501b35edae86d))
* enhance VirtualTable to have expression as value ([#711](https://github.com/substrait-io/substrait/issues/711)) ([954bcbc](https://github.com/substrait-io/substrait/commit/954bcbc1a3eeabb696e7dc10721b85e1f475ecfd))
* specify row_number start ([#722](https://github.com/substrait-io/substrait/issues/722)) ([#723](https://github.com/substrait-io/substrait/issues/723)) ([a0388ff](https://github.com/substrait-io/substrait/commit/a0388ff69a83aa4addf51236b77bf275b3647590))

## [0.57.1](https://github.com/substrait-io/substrait/compare/v0.57.0...v0.57.1) (2024-10-06)

### Bug Fixes
Expand Down
17 changes: 13 additions & 4 deletions core.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,20 @@ package substrait

import "embed"

// Add all directories which should be exposed in below
//
//go:embed extensions/*
var substraitFS embed.FS
var substraitExtensionsFS embed.FS

func GetSubstraitFS() embed.FS {
return substraitFS
return substraitExtensionsFS
}

func GetSubstraitExtensionsFS() embed.FS {
return substraitExtensionsFS
}

//go:embed tests/cases/*/*.test
var substraitTestsFS embed.FS

func GetSubstraitTestsFS() embed.FS {
return substraitTestsFS
}
47 changes: 47 additions & 0 deletions core_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package substrait

import (
"embed"
"io/fs"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestGetSubstraitExtensionsFS(t *testing.T) {
fsArr := []embed.FS{GetSubstraitExtensionsFS(), GetSubstraitFS()}
for _, got := range fsArr {
filePaths, err := ListFiles(got, ".")
require.NoError(t, err)
assert.Greater(t, len(filePaths), 15)
assert.Contains(t, filePaths, "extensions/functions_arithmetic.yaml")
assert.Contains(t, filePaths, "extensions/functions_arithmetic_decimal.yaml")
assert.Contains(t, filePaths, "extensions/functions_datetime.yaml")
}
}

func TestGetSubstraitTestsFS(t *testing.T) {
got := GetSubstraitTestsFS()
filePaths, err := ListFiles(got, ".")
require.NoError(t, err)
assert.Greater(t, len(filePaths), 3)
assert.Contains(t, filePaths, "tests/cases/arithmetic/add.test")
assert.Contains(t, filePaths, "tests/cases/arithmetic/max.test")
assert.Contains(t, filePaths, "tests/cases/arithmetic_decimal/power.test")
assert.Contains(t, filePaths, "tests/cases/datetime/lt_datetime.test")
}

func ListFiles(embedFs embed.FS, root string) ([]string, error) {
var files []string
err := fs.WalkDir(embedFs, root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
files = append(files, path)
}
return nil
})
return files, err
}
2 changes: 1 addition & 1 deletion extensions/functions_arithmetic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1648,7 +1648,7 @@ aggregate_functions:

window_functions:
- name: "row_number"
description: "the number of the current row within its partition."
description: "the number of the current row within its partition, starting at 1"
impls:
- args: []
nullability: DECLARED_OUTPUT
Expand Down
8 changes: 8 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
module github.com/substrait-io/substrait

go 1.22.0

require github.com/stretchr/testify v1.9.0

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
113 changes: 113 additions & 0 deletions grammar/FuncTestCaseLexer.g4
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
lexer grammar FuncTestCaseLexer;

import SubstraitLexer;

options {
caseInsensitive = true;
}

Whitespace : [ \t\n\r]+ -> channel(HIDDEN) ;

TripleHash: '###';
SubstraitScalarTest: 'SUBSTRAIT_SCALAR_TEST';
SubstraitAggregateTest: 'SUBSTRAIT_AGGREGATE_TEST';
SubstraitInclude: 'SUBSTRAIT_INCLUDE';

FormatVersion
: 'v' DIGIT+ ('.' DIGIT+)?
;

DescriptionLine
: '# ' ~[\r\n]* '\r'? '\n'
;

Define: 'DEFINE';
ErrorResult: '<!ERROR>';
UndefineResult: '<!UNDEFINED>';
Overflow: 'OVERFLOW';
Rounding: 'ROUNDING';
Error: 'ERROR';
Saturate: 'SATURATE';
Silent: 'SILENT';
TieToEven: 'TIE_TO_EVEN';
NaN: 'NAN';
AcceptNulls: 'ACCEPT_NULLS';
IgnoreNulls: 'IGNORE_NULLS';
NullHandling: 'NULL_HANDLING';
SpacesOnly: 'SPACES_ONLY';
Truncate: 'TRUNCATE';

IntegerLiteral
: [+-]? Int
;

DecimalLiteral
: [+-]? [0-9]+ ('.' [0-9]+)?
;

FloatLiteral
: [+-]? [0-9]+ ('.' [0-9]*)? ( 'E' [+-]? [0-9]+ )?
| [+-]? 'inf'
| 'snan'
;

BooleanLiteral
: 'true' | 'false'
;

fragment FourDigits: [0-9][0-9][0-9][0-9];
fragment TwoDigits: [0-9][0-9];

TimestampTzLiteral
: '\'' FourDigits '-' TwoDigits '-' TwoDigits 'T' TwoDigits ':' TwoDigits ':' TwoDigits ( '.' [0-9]+ )?
[+-] TwoDigits ':' TwoDigits '\''
;

TimestampLiteral
: '\'' FourDigits '-' TwoDigits '-' TwoDigits 'T' TwoDigits ':' TwoDigits ':' TwoDigits ( '.' [0-9]+ )? '\''
;

TimeLiteral
: '\'' TwoDigits ':' TwoDigits ':' TwoDigits ( '.' [0-9]+ )? '\''
;

DateLiteral
: '\'' FourDigits '-' TwoDigits '-' TwoDigits '\''
;

PeriodPrefix: 'P';
TimePrefix: 'T';
YearPrefix: 'Y';
MSuffix: 'M'; // used for both months and minutes
DaySuffix: 'D';
HourSuffix: 'H';
SecondSuffix: 'S';
FractionalSecondSuffix: 'F';
OAngleBracket: Lt;
CAngleBracket: Gt;

IntervalYearLiteral
: '\'' PeriodPrefix IntegerLiteral YearPrefix (IntegerLiteral MSuffix)? '\''
| '\'' PeriodPrefix IntegerLiteral MSuffix '\''
;

IntervalDayLiteral
: '\'' PeriodPrefix IntegerLiteral DaySuffix (TimePrefix TimeInterval)? '\''
| '\'' PeriodPrefix TimePrefix TimeInterval '\''
;

fragment TimeInterval
: IntegerLiteral HourSuffix (IntegerLiteral MSuffix)? (DecimalLiteral SecondSuffix)?
| IntegerLiteral MSuffix (DecimalLiteral SecondSuffix)?
| DecimalLiteral SecondSuffix
;

NullLiteral: 'null';

StringLiteral
: '\'' ('\\' . | '\'\'' | ~['\\])* '\''
;

ColumnName
: 'COL' Int
;
Loading

0 comments on commit 19d8c55

Please sign in to comment.