diff --git a/embedded/sql/catalog.go b/embedded/sql/catalog.go index 8865e245ca..bb224b2373 100644 --- a/embedded/sql/catalog.go +++ b/embedded/sql/catalog.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "encoding/binary" + "encoding/json" "errors" "fmt" "io" @@ -409,8 +410,8 @@ func (catlg *Catalog) newTable(name string, colsSpec map[uint32]*ColSpec, maxCol continue } - if cs.colName == revCol { - return nil, fmt.Errorf("%w(%s)", ErrReservedWord, revCol) + if isReservedCol(cs.colName) { + return nil, fmt.Errorf("%w(%s)", ErrReservedWord, cs.colName) } _, colExists := table.colsByName[cs.colName] @@ -533,8 +534,8 @@ func (t *Table) newIndex(unique bool, colIDs []uint32) (index *Index, err error) } func (t *Table) newColumn(spec *ColSpec) (*Column, error) { - if spec.colName == revCol { - return nil, fmt.Errorf("%w(%s)", ErrReservedWord, revCol) + if isReservedCol(spec.colName) { + return nil, fmt.Errorf("%w(%s)", ErrReservedWord, spec.colName) } if spec.autoIncrement { @@ -597,8 +598,8 @@ func (ctlg *Catalog) renameTable(oldName, newName string) (*Table, error) { } func (t *Table) renameColumn(oldName, newName string) (*Column, error) { - if newName == revCol { - return nil, fmt.Errorf("%w(%s)", ErrReservedWord, revCol) + if isReservedCol(newName) { + return nil, fmt.Errorf("%w(%s)", ErrReservedWord, newName) } if oldName == newName { @@ -1006,16 +1007,17 @@ func unmapColSpec(prefix, mkey []byte) (dbID, tableID, colID uint32, colType SQL } func asType(t string) (SQLValueType, error) { - if t == IntegerType || - t == Float64Type || - t == BooleanType || - t == VarcharType || - t == UUIDType || - t == BLOBType || - t == TimestampType { + switch t { + case IntegerType, + Float64Type, + BooleanType, + VarcharType, + UUIDType, + BLOBType, + TimestampType, + JSONType: return t, nil } - return t, ErrCorruptedData } @@ -1304,12 +1306,37 @@ func EncodeRawValueAsKey(val interface{}, colType SQLValueType, maxLen int) ([]b return nil, 0, ErrInvalidValue } +func getEncodeRawValue(val TypedValue, colType SQLValueType) (interface{}, error) { + if colType != JSONType || val.Type() == JSONType { + return val.RawValue(), nil + } + + if val.Type() != VarcharType { + return nil, fmt.Errorf("%w: invalid json value", ErrInvalidValue) + } + s, _ := val.RawValue().(string) + + raw := json.RawMessage(s) + if !json.Valid(raw) { + return nil, fmt.Errorf("%w: invalid json value", ErrInvalidValue) + } + return raw, nil +} + func EncodeValue(val TypedValue, colType SQLValueType, maxLen int) ([]byte, error) { - return EncodeRawValue(val.RawValue(), colType, maxLen, false) + v, err := getEncodeRawValue(val, colType) + if err != nil { + return nil, err + } + return EncodeRawValue(v, colType, maxLen, false) } func EncodeNullableValue(val TypedValue, colType SQLValueType, maxLen int) ([]byte, error) { - return EncodeRawValue(val.RawValue(), colType, maxLen, true) + v, err := getEncodeRawValue(val, colType) + if err != nil { + return nil, err + } + return EncodeRawValue(v, colType, maxLen, true) } // EncodeRawValue encode a value in a byte format. This is the internal binary representation of a value. Can be decoded with DecodeValue. @@ -1402,6 +1429,22 @@ func EncodeRawValue(val interface{}, colType SQLValueType, maxLen int, nullable return encv[:], nil } + case JSONType: + rawJson, ok := val.(json.RawMessage) + if !ok { + data, err := json.Marshal(val) + if err != nil { + return nil, err + } + rawJson = data + } + + // len(v) + v + encv := make([]byte, EncLenLen+len(rawJson)) + binary.BigEndian.PutUint32(encv[:], uint32(len(rawJson))) + copy(encv[EncLenLen:], rawJson) + + return encv[:], nil case UUIDType: { uuidVal, ok := convVal.(uuid.UUID) @@ -1519,6 +1562,16 @@ func decodeValue(b []byte, colType SQLValueType, nullable bool) (TypedValue, int return &Blob{val: v}, voff, nil } + case JSONType: + { + v := b[voff : voff+vlen] + voff += vlen + + var val interface{} + err = json.Unmarshal(v, &val) + + return &JSON{val: val}, voff, err + } case UUIDType: { if vlen != 16 { diff --git a/embedded/sql/engine.go b/embedded/sql/engine.go index e6850465c3..ba33f73d9b 100644 --- a/embedded/sql/engine.go +++ b/embedded/sql/engine.go @@ -55,6 +55,7 @@ var ErrNewColumnMustBeNullable = errors.New("new column must be nullable") var ErrIndexAlreadyExists = errors.New("index already exists") var ErrMaxNumberOfColumnsInIndexExceeded = errors.New("number of columns in multi-column index exceeded") var ErrIndexNotFound = errors.New("index not found") +var ErrCannotIndexJson = errors.New("cannot index column of type json") var ErrInvalidNumberOfValues = errors.New("invalid number of values provided") var ErrInvalidValue = errors.New("invalid value provided") var ErrInferredMultipleTypes = errors.New("inferred multiple types") @@ -89,6 +90,7 @@ var ErrAlreadyClosed = store.ErrAlreadyClosed var ErrAmbiguousSelector = errors.New("ambiguous selector") var ErrUnsupportedCast = fmt.Errorf("%w: unsupported cast", ErrInvalidValue) var ErrColumnMismatchInUnionStmt = errors.New("column mismatch in union statement") +var ErrInvalidTxMetadata = errors.New("invalid transaction metadata") var MaxKeyLen = 512 @@ -105,8 +107,8 @@ type Engine struct { sortBufferSize int autocommit bool lazyIndexConstraintValidation bool - - multidbHandler MultiDBHandler + parseTxMetadata func([]byte) (map[string]interface{}, error) + multidbHandler MultiDBHandler } type MultiDBHandler interface { @@ -146,6 +148,7 @@ func NewEngine(st *store.ImmuStore, opts *Options) (*Engine, error) { sortBufferSize: opts.sortBufferSize, autocommit: opts.autocommit, lazyIndexConstraintValidation: opts.lazyIndexConstraintValidation, + parseTxMetadata: opts.parseTxMetadata, multidbHandler: opts.multidbHandler, } diff --git a/embedded/sql/engine_test.go b/embedded/sql/engine_test.go index 1e059b10c6..dc24ea724e 100644 --- a/embedded/sql/engine_test.go +++ b/embedded/sql/engine_test.go @@ -19,6 +19,7 @@ package sql import ( "context" "encoding/hex" + "encoding/json" "errors" "fmt" "math" @@ -2786,6 +2787,350 @@ func TestQuery(t *testing.T) { require.NoError(t, err) } +func TestJSON(t *testing.T) { + opts := store.DefaultOptions().WithMultiIndexing(true) + opts.WithIndexOptions(opts.IndexOpts.WithMaxActiveSnapshots(1)) + + st, err := store.Open(t.TempDir(), opts) + require.NoError(t, err) + defer closeStore(t, st) + + engine, err := NewEngine(st, DefaultOptions().WithPrefix(sqlPrefix)) + require.NoError(t, err) + + _, _, err = engine.Exec( + context.Background(), nil, + ` + CREATE TABLE tbl_with_json ( + id INTEGER AUTO_INCREMENT, + json_data JSON NOT NULL, + + PRIMARY KEY(id) + )`, nil) + require.NoError(t, err) + + _, _, err = engine.Exec( + context.Background(), + nil, + `INSERT INTO tbl_with_json(json_data) VALUES ('invalid json value')`, + nil, + ) + require.ErrorIs(t, err, ErrInvalidValue) + + _, _, err = engine.Exec( + context.Background(), + nil, + `INSERT INTO tbl_with_json(json_data) VALUES (10)`, + nil, + ) + require.ErrorIs(t, err, ErrInvalidValue) + + n := 100 + for i := 0; i < n; i++ { + data := fmt.Sprintf( + `{"usr": {"name": "%s", "active": %t, "details": {"age": %d, "city": "%s"}, "perms": ["r", "w"]}}`, + fmt.Sprintf("name%d", i+1), + i%2 == 0, + i+1, + fmt.Sprintf("city%d", i+1), + ) + + _, _, err = engine.Exec( + context.Background(), + nil, + fmt.Sprintf(`INSERT INTO tbl_with_json(json_data) VALUES ('%s')`, data), + nil, + ) + require.NoError(t, err) + } + + t.Run("apply -> operator on non JSON column", func(t *testing.T) { + _, err := engine.queryAll( + context.Background(), + nil, + "SELECT id->'name' FROM tbl_with_json", + nil, + ) + require.ErrorContains(t, err, "-> operator cannot be applied on column of type INTEGER") + }) + + t.Run("filter json fields", func(t *testing.T) { + t.Run("filter boolean value", func(t *testing.T) { + rows, err := engine.queryAll( + context.Background(), + nil, + ` + SELECT json_data->'usr' + FROM tbl_with_json + WHERE json_data->'usr'->'active' = TRUE + `, + nil, + ) + require.NoError(t, err) + require.Len(t, rows, n/2) + + for i, row := range rows { + usr, _ := row.ValuesBySelector[EncodeSelector("", "tbl_with_json", "json_data->usr")].RawValue().(map[string]interface{}) + + require.Equal(t, map[string]interface{}{ + "name": fmt.Sprintf("name%d", (2*i + 1)), + "active": true, + "details": map[string]interface{}{ + "age": float64((2*i + 1)), + "city": fmt.Sprintf("city%d", (2*i + 1)), + }, + "perms": []interface{}{ + "r", "w", + }, + }, usr) + } + }) + + t.Run("filter numeric value", func(t *testing.T) { + rows, err := engine.queryAll( + context.Background(), + nil, + ` + SELECT json_data->'usr'->'name' + FROM tbl_with_json + WHERE json_data->'usr'->'details'->'age' + 1 >= 52 + `, + nil, + ) + require.NoError(t, err) + require.Len(t, rows, n/2) + + for i, row := range rows { + name := row.ValuesByPosition[0].RawValue() + require.Equal(t, name, fmt.Sprintf("name%d", 51+i)) + } + }) + + t.Run("filter varchar value", func(t *testing.T) { + rows, err := engine.queryAll( + context.Background(), + nil, + ` + SELECT json_data->'usr'->'name' + FROM tbl_with_json + WHERE json_data->'usr'->'name' LIKE '^name.*' AND json_data->'usr'->'perms'->'0' = 'r' + `, + nil, + ) + require.NoError(t, err) + require.Len(t, rows, n) + + for i, row := range rows { + name := row.ValuesByPosition[0].RawValue() + require.Equal(t, name, fmt.Sprintf("name%d", i+1)) + } + }) + }) + + t.Run("order by json field", func(t *testing.T) { + _, err := engine.queryAll( + context.Background(), + nil, + ` + SELECT json_data + FROM tbl_with_json + ORDER BY json_data + `, + nil, + ) + require.ErrorIs(t, err, ErrNotComparableValues) + + rows, err := engine.queryAll( + context.Background(), + nil, + ` + SELECT json_data->'usr', json_data->'usr'->'details'->'age' as age, json_data->'usr'->'details'->'city' as city, json_data->'usr'->'name' as name + FROM tbl_with_json + ORDER BY json_data->'usr'->'details'->'age' DESC + `, + nil, + ) + require.NoError(t, err) + require.Len(t, rows, n) + + for i, row := range rows { + usr, _ := row.ValuesBySelector[EncodeSelector("", "tbl_with_json", "json_data->usr")].RawValue().(map[string]interface{}) + name, _ := row.ValuesBySelector[EncodeSelector("", "tbl_with_json", "name")].RawValue().(string) + age, _ := row.ValuesBySelector[EncodeSelector("", "tbl_with_json", "age")].RawValue().(float64) + city, _ := row.ValuesBySelector[EncodeSelector("", "tbl_with_json", "city")].RawValue().(string) + + require.Equal(t, map[string]interface{}{ + "name": name, + "active": (n-1-i)%2 == 0, + "details": map[string]interface{}{ + "age": age, + "city": city, + }, + "perms": []interface{}{"r", "w"}, + }, usr) + + require.Equal(t, fmt.Sprintf("name%d", n-i), name) + require.Equal(t, float64(n-i), age) + require.Equal(t, fmt.Sprintf("city%d", n-i), city) + } + }) + + t.Run("test join on json field", func(t *testing.T) { + _, _, err = engine.Exec(context.Background(), nil, "CREATE TABLE table1(id INTEGER AUTO_INCREMENT, value VARCHAR, PRIMARY KEY(id))", nil) + require.NoError(t, err) + + for i := 0; i < 10; i++ { + _, _, err = engine.Exec(context.Background(), nil, "INSERT INTO table1(value) VALUES (@name)", map[string]interface{}{"name": fmt.Sprintf("name%d", i+1)}) + require.NoError(t, err) + } + + rows, err := engine.queryAll( + context.Background(), + nil, + "SELECT table1.value, json_data->'usr'->'name' FROM tbl_with_json JOIN table1 ON table1.value = tbl_with_json.json_data->'usr'->'name' ORDER BY table1.id", + nil, + ) + require.NoError(t, err) + require.Len(t, rows, 10) + + for i, row := range rows { + require.Len(t, row.ValuesByPosition, 2) + require.Equal(t, fmt.Sprintf("name%d", i+1), row.ValuesByPosition[0].RawValue()) + require.Equal(t, row.ValuesByPosition[0].RawValue(), row.ValuesByPosition[1].RawValue()) + } + }) + + _, _, err = engine.Exec(context.Background(), nil, "DELETE FROM tbl_with_json", nil) + require.NoError(t, err) + + randJson := func(src *rand.Rand) interface{} { + switch src.Intn(6) { + case 0: + return src.Float64() + case 1: + return fmt.Sprintf("string%d", src.Int63()) + case 2: + return src.Int()%2 == 0 + case 3: + return map[string]interface{}{ + "test": "value", + } + case 4: + return []interface{}{"test", true, 10.5} + } + return nil + } + + seed := time.Now().UnixNano() + src := rand.New(rand.NewSource(seed)) + for i := 0; i < n; i++ { + data := randJson(src) + + jsonData, err := json.Marshal(data) + require.NoError(t, err) + + _, _, err = engine.Exec( + context.Background(), + nil, + "INSERT INTO tbl_with_json(json_data) VALUES (@data)", + map[string]interface{}{"data": string(jsonData)}, + ) + require.NoError(t, err) + } + + t.Run("lookup field", func(t *testing.T) { + rows, err := engine.queryAll( + context.Background(), + nil, + "SELECT json_data, json_data->'test' FROM tbl_with_json", + nil, + ) + require.NoError(t, err) + require.Len(t, rows, n) + + for _, row := range rows { + data := row.ValuesByPosition[0].RawValue() + value := row.ValuesByPosition[1].RawValue() + if _, isObject := data.(map[string]interface{}); isObject { + require.Equal(t, "value", row.ValuesByPosition[1].RawValue()) + } else { + require.Nil(t, value) + } + } + }) + + t.Run("query json with mixed types", func(t *testing.T) { + rows, err := engine.queryAll( + context.Background(), + nil, + "SELECT json_data FROM tbl_with_json", + nil, + ) + require.NoError(t, err) + require.Len(t, rows, n) + + stringValues := 0 + + src := rand.New(rand.NewSource(seed)) + for _, row := range rows { + s := row.ValuesByPosition[0].RawValue() + require.Equal(t, randJson(src), s) + + if _, ok := s.(string); ok { + stringValues++ + } + } + + rows, err = engine.queryAll( + context.Background(), + nil, + "SELECT COUNT(*) FROM tbl_with_json WHERE json_typeof(json_data) = 'STRING'", + nil, + ) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, rows[0].ValuesByPosition[0].RawValue(), int64(stringValues)) + }) + + t.Run("update json data", func(t *testing.T) { + _, _, err = engine.Exec( + context.Background(), + nil, + fmt.Sprintf(`UPDATE tbl_with_json SET json_data = '%d' WHERE json_typeof(json_data) = 'STRING'`, rand.Int63()), + nil, + ) + require.NoError(t, err) + + rows, err := engine.queryAll( + context.Background(), + nil, + "SELECT COUNT(*) FROM tbl_with_json WHERE json_typeof(json_data) = 'STRING'", + nil, + ) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Zero(t, rows[0].ValuesByPosition[0].RawValue()) + }) + + t.Run("cannot index json column", func(t *testing.T) { + _, _, err = engine.Exec( + context.Background(), + nil, + "CREATE INDEX ON tbl_with_json(json_data);", nil) + require.ErrorIs(t, err, ErrCannotIndexJson) + + _, _, err = engine.Exec( + context.Background(), nil, + ` + CREATE TABLE test ( + json_data JSON NOT NULL, + + PRIMARY KEY(json_data) + )`, nil) + require.ErrorIs(t, err, ErrCannotIndexJson) + }) +} + func TestQueryCornerCases(t *testing.T) { opts := store.DefaultOptions().WithMultiIndexing(true) opts.WithIndexOptions(opts.IndexOpts.WithMaxActiveSnapshots(1)) @@ -2813,7 +3158,6 @@ func TestQueryCornerCases(t *testing.T) { require.NoError(t, err) t.Run("run out of snapshots", func(t *testing.T) { - // Get one tx that takes the snapshot tx, err := engine.NewTx(context.Background(), DefaultTxOptions()) require.NoError(t, err) @@ -8168,3 +8512,86 @@ func (t *BrokenCatalogTestSuite) TestErrorDroppedPrimaryIndexColumn() { err = c.load(context.Background(), tx.tx) t.Require().ErrorIs(err, ErrColumnDoesNotExist) } + +func TestQueryTxMetadata(t *testing.T) { + opts := store.DefaultOptions().WithMultiIndexing(true) + opts.WithIndexOptions(opts.IndexOpts.WithMaxActiveSnapshots(1)) + + st, err := store.Open(t.TempDir(), opts) + require.NoError(t, err) + defer closeStore(t, st) + + engine, err := NewEngine(st, + DefaultOptions().WithPrefix(sqlPrefix).WithParseTxMetadataFunc(func(b []byte) (map[string]interface{}, error) { + var md map[string]interface{} + err := json.Unmarshal(b, &md) + return md, err + }), + ) + require.NoError(t, err) + + _, _, err = engine.Exec( + context.Background(), nil, + ` + CREATE TABLE mytbl ( + id INTEGER AUTO_INCREMENT, + + PRIMARY KEY(id) + )`, nil) + require.NoError(t, err) + + for i := 0; i < 10; i++ { + extra, err := json.Marshal(map[string]interface{}{ + "n": i + 1, + }) + require.NoError(t, err) + + txOpts := DefaultTxOptions().WithExtra(extra) + tx, err := engine.NewTx(context.Background(), txOpts) + require.NoError(t, err) + + _, _, err = engine.Exec( + context.Background(), + tx, + fmt.Sprintf("INSERT INTO mytbl(id) VALUES (%d)", i+1), + nil, + ) + require.NoError(t, err) + } + + rows, err := engine.queryAll( + context.Background(), + nil, + "SELECT _tx_metadata->'n' FROM mytbl", + nil, + ) + require.NoError(t, err) + require.Len(t, rows, 10) + + for i, row := range rows { + n := row.ValuesBySelector[EncodeSelector("", "mytbl", "_tx_metadata->n")].RawValue() + require.Equal(t, float64(i+1), n) + } + + engine.parseTxMetadata = nil + + _, err = engine.queryAll( + context.Background(), + nil, + "SELECT _tx_metadata->'n' FROM mytbl", + nil, + ) + require.ErrorContains(t, err, "unable to parse tx metadata") + + engine.parseTxMetadata = func(b []byte) (map[string]interface{}, error) { + return nil, fmt.Errorf("parse error") + } + + _, err = engine.queryAll( + context.Background(), + nil, + "SELECT _tx_metadata->'n' FROM mytbl", + nil, + ) + require.ErrorIs(t, err, ErrInvalidTxMetadata) +} diff --git a/embedded/sql/file_sort.go b/embedded/sql/file_sort.go index 66a71659f4..b15fa6c16d 100644 --- a/embedded/sql/file_sort.go +++ b/embedded/sql/file_sort.go @@ -33,7 +33,7 @@ type sortedChunk struct { type fileSorter struct { colPosBySelector map[string]int colTypes []string - cmp func(r1, r2 Tuple) int + cmp func(r1, r2 *Row) (int, error) tx *SQLTx sortBufSize int @@ -64,7 +64,9 @@ func (s *fileSorter) update(r *Row) error { func (s *fileSorter) finalize() (resultReader, error) { if s.nextIdx > 0 { - s.sortBuffer() + if err := s.sortBuffer(); err != nil { + return nil, err + } } // result rows are all in memory @@ -98,12 +100,16 @@ func (s *fileSorter) mergeAllChunks() (resultReader, error) { rbuf := &bufio.Reader{} lr := &fileRowReader{ - colTypes: s.colTypes, - reader: lbuf, + colPosBySelector: s.colPosBySelector, + colTypes: s.colTypes, + reader: lbuf, + reuseRow: true, } rr := &fileRowReader{ - colTypes: s.colTypes, - reader: rbuf, + colPosBySelector: s.colPosBySelector, + colTypes: s.colTypes, + reader: rbuf, + reuseRow: true, } chunks := s.chunksToMerge @@ -169,12 +175,12 @@ func (s *fileSorter) mergeAllChunks() (resultReader, error) { func (s *fileSorter) mergeChunks(lr, rr *fileRowReader, writer io.Writer) error { var err error var lrAtEOF bool - var t1, t2 Tuple + var r1, r2 *Row for { - if t1 == nil { - t1, err = lr.ReadValues() - if err == io.EOF { + if r1 == nil { + r1, err = lr.Read() + if err == ErrNoMoreRows { lrAtEOF = true break } @@ -184,9 +190,9 @@ func (s *fileSorter) mergeChunks(lr, rr *fileRowReader, writer io.Writer) error } } - if t2 == nil { - t2, err = rr.ReadValues() - if err == io.EOF { + if r2 == nil { + r2, err = rr.Read() + if err == ErrNoMoreRows { break } @@ -196,15 +202,20 @@ func (s *fileSorter) mergeChunks(lr, rr *fileRowReader, writer io.Writer) error } var rawData []byte - if s.cmp(t1, t2) < 0 { + res, err := s.cmp(r1, r2) + if err != nil { + return err + } + + if res < 0 { rawData = lr.rowBuf.Bytes() - t1 = nil + r1 = nil } else { rawData = rr.rowBuf.Bytes() - t2 = nil + r2 = nil } - _, err := writer.Write(rawData) + _, err = writer.Write(rawData) if err != nil { return err } @@ -248,13 +259,15 @@ type fileRowReader struct { colTypes []SQLValueType reader io.Reader rowBuf bytes.Buffer + row *Row + reuseRow bool } -func (r *fileRowReader) ReadValues() ([]TypedValue, error) { +func (r *fileRowReader) readValues(out []TypedValue) error { var size uint16 err := binary.Read(r.reader, binary.BigEndian, &size) if err != nil { - return nil, err + return err } r.rowBuf.Reset() @@ -263,15 +276,17 @@ func (r *fileRowReader) ReadValues() ([]TypedValue, error) { _, err = io.CopyN(&r.rowBuf, r.reader, int64(size)) if err != nil { - return nil, err + return err } data := r.rowBuf.Bytes() - return decodeValues(data[2:], r.colTypes) + return decodeValues(data[2:], r.colTypes, out) } func (r *fileRowReader) Read() (*Row, error) { - values, err := r.ReadValues() + row := r.getRow() + + err := r.readValues(row.ValuesByPosition) if err == io.EOF { return nil, ErrNoMoreRows } @@ -279,48 +294,60 @@ func (r *fileRowReader) Read() (*Row, error) { return nil, err } - valuesBySelector := make(map[string]TypedValue) for sel, pos := range r.colPosBySelector { - valuesBySelector[sel] = values[pos] - } - - row := &Row{ - ValuesByPosition: values, - ValuesBySelector: valuesBySelector, + row.ValuesBySelector[sel] = row.ValuesByPosition[pos] } return row, nil } -func decodeValues(data []byte, colTypes []SQLValueType) ([]TypedValue, error) { - values := make([]TypedValue, len(colTypes)) +func (r *fileRowReader) getRow() *Row { + row := r.row + if row == nil || !r.reuseRow { + row = &Row{ + ValuesByPosition: make([]TypedValue, len(r.colPosBySelector)), + ValuesBySelector: make(map[string]TypedValue, len(r.colPosBySelector)), + } + r.row = row + } + return row +} +func decodeValues(data []byte, colTypes []SQLValueType, out []TypedValue) error { var voff int for i, col := range colTypes { v, n, err := DecodeNullableValue(data[voff:], col) if err != nil { - return nil, err + return err } voff += n - values[i] = v + out[i] = v } - return values, nil + return nil } func (s *fileSorter) sortAndFlushBuffer() error { - s.sortBuffer() + if err := s.sortBuffer(); err != nil { + return err + } return s.flushBuffer() } -func (s *fileSorter) sortBuffer() { +func (s *fileSorter) sortBuffer() error { buf := s.sortBuf[:s.nextIdx] + var outErr error sort.Slice(buf, func(i, j int) bool { r1 := buf[i] r2 := buf[j] - return s.cmp(r1.ValuesByPosition, r2.ValuesByPosition) < 0 + res, err := s.cmp(r1, r2) + if err != nil { + outErr = err + } + return res < 0 }) + return outErr } func (s *fileSorter) flushBuffer() error { diff --git a/embedded/sql/json_type.go b/embedded/sql/json_type.go new file mode 100644 index 0000000000..1ac19565a9 --- /dev/null +++ b/embedded/sql/json_type.go @@ -0,0 +1,211 @@ +package sql + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) + +const ( + JSONTypeNumber = "NUMBER" + JSONTypeBool = "BOOL" + JSONTypeString = "STRING" + JSONTypeArray = "ARRAY" + JSONTypeObject = "OBJECT" + JSONTypeNull = "NULL" +) + +type JSON struct { + val interface{} +} + +func NewJson(val interface{}) *JSON { + return &JSON{val: val} +} + +func (v *JSON) Type() SQLValueType { + return JSONType +} + +func (v *JSON) IsNull() bool { + return false +} + +func (v *JSON) inferType(cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) (SQLValueType, error) { + return JSONType, nil +} + +func (v *JSON) requiresType(t SQLValueType, cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) error { + ok := t == JSONType + switch t { + case IntegerType, Float64Type: + _, isInt := v.val.(int64) + _, isFloat := v.val.(float64) + ok = isInt || (isFloat && t == Float64Type) + case VarcharType: + _, ok = v.val.(string) + case BooleanType: + _, ok = v.val.(bool) + case AnyType: + ok = v.val == nil + } + + if !ok { + return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, JSONType, t) + } + return nil +} + +func (v *JSON) substitute(params map[string]interface{}) (ValueExp, error) { + return v, nil +} + +func (v *JSON) reduce(tx *SQLTx, row *Row, implicitTable string) (TypedValue, error) { + return v, nil +} + +func (v *JSON) reduceSelectors(row *Row, implicitTable string) ValueExp { + return v +} + +func (v *JSON) isConstant() bool { + return true +} + +func (v *JSON) selectorRanges(table *Table, asTable string, params map[string]interface{}, rangesByColID map[uint32]*typedValueRange) error { + return nil +} + +func (v *JSON) RawValue() interface{} { + return v.val +} + +func (v *JSON) Compare(val TypedValue) (int, error) { + tv, ok := v.castToTypedValue() + if !ok { + return -1, fmt.Errorf("%w: comparison not defined for JSON %s", ErrNotComparableValues, v.primitiveType()) + } + + if val.Type() != JSONType { + return tv.Compare(val) + } + + res, err := val.Compare(tv) + return -res, err +} + +func (v *JSON) primitiveType() string { + switch v.val.(type) { + case int64, float64: + return JSONTypeNumber + case string: + return JSONTypeString + case bool: + return JSONTypeBool + case nil: + return JSONTypeNull + case []interface{}: + return JSONTypeArray + } + return JSONTypeObject +} + +func (v *JSON) castToTypedValue() (TypedValue, bool) { + var tv TypedValue + switch val := v.val.(type) { + case int64: + tv = NewInteger(val) + case string: + tv = NewVarchar(val) + case float64: + tv = NewFloat64(val) + case bool: + tv = NewBool(val) + case nil: + tv = NewNull(JSONType) + default: + return nil, false + } + return tv, true +} + +func (v *JSON) String() string { + data, _ := json.Marshal(v.val) + return string(data) +} + +func (v *JSON) lookup(fields []string) TypedValue { + currVal := v.val + for i, field := range fields { + switch cv := currVal.(type) { + case map[string]interface{}: + v, hasField := cv[field] + if !hasField || (v == nil && i < len(field)-1) { + return NewNull(AnyType) + } + currVal = v + + if currVal == nil { + break + } + case []interface{}: + idx, err := strconv.ParseInt(field, 10, 64) + if err != nil || idx < 0 || idx >= int64(len(cv)) { + return NewNull(AnyType) + } + currVal = cv[idx] + default: + return NewNull(AnyType) + } + } + return NewJson(currVal) +} + +type JSONSelector struct { + *ColSelector + fields []string +} + +func (sel *JSONSelector) substitute(params map[string]interface{}) (ValueExp, error) { + return sel, nil +} + +func (v *JSONSelector) alias() string { + if v.ColSelector.as != "" { + return v.ColSelector.as + } + return v.string() +} + +func (v *JSONSelector) resolve(implicitTable string) (string, string, string) { + aggFn, table, _ := v.ColSelector.resolve(implicitTable) + return aggFn, table, v.string() +} + +func (v *JSONSelector) string() string { + return fmt.Sprintf("%s->%s", v.ColSelector.col, strings.Join(v.fields, "->")) +} + +func (sel *JSONSelector) reduce(tx *SQLTx, row *Row, implicitTable string) (TypedValue, error) { + val, err := sel.ColSelector.reduce(tx, row, implicitTable) + if err != nil { + return nil, err + } + + jsonVal, ok := val.(*JSON) + if !ok { + return val, fmt.Errorf("-> operator cannot be applied on column of type %s", val.Type()) + } + return jsonVal.lookup(sel.fields), nil +} + +func (sel *JSONSelector) reduceSelectors(row *Row, implicitTable string) ValueExp { + val := sel.ColSelector.reduceSelectors(row, implicitTable) + + jsonVal, ok := val.(*JSON) + if !ok { + return sel + } + return jsonVal.lookup(sel.fields) +} diff --git a/embedded/sql/num_operator.go b/embedded/sql/num_operator.go index 652c9069c3..8d4731e7f3 100644 --- a/embedded/sql/num_operator.go +++ b/embedded/sql/num_operator.go @@ -22,7 +22,6 @@ func applyNumOperator(op NumOperator, vl, vr TypedValue) (TypedValue, error) { if vl.Type() == Float64Type || vr.Type() == Float64Type { return applyNumOperatorFloat64(op, vl, vr) } - return applyNumOperatorInteger(op, vl, vr) } diff --git a/embedded/sql/options.go b/embedded/sql/options.go index ed0edbf36f..3939f27655 100644 --- a/embedded/sql/options.go +++ b/embedded/sql/options.go @@ -33,6 +33,7 @@ type Options struct { distinctLimit int autocommit bool lazyIndexConstraintValidation bool + parseTxMetadata func([]byte) (map[string]interface{}, error) multidbHandler MultiDBHandler } @@ -92,3 +93,8 @@ func (opts *Options) WithSortBufferSize(size int) *Options { opts.sortBufferSize = size return opts } + +func (opts *Options) WithParseTxMetadataFunc(parseFunc func([]byte) (map[string]interface{}, error)) *Options { + opts.parseTxMetadata = parseFunc + return opts +} diff --git a/embedded/sql/parser.go b/embedded/sql/parser.go index a2922a8eb0..11afcf3f80 100644 --- a/embedded/sql/parser.go +++ b/embedded/sql/parser.go @@ -118,6 +118,7 @@ var types = map[string]SQLValueType{ "BLOB": BLOBType, "TIMESTAMP": TimestampType, "FLOAT": Float64Type, + "JSON": JSONType, } var aggregateFns = map[string]AggregateFn{ @@ -276,6 +277,11 @@ func (l *lexer) Lex(lval *yySymType) int { return STMT_SEPARATOR } + if ch == '-' && l.r.nextChar == '>' { + l.r.ReadByte() + return ARROW + } + if isBLOBPrefix(ch) && isQuote(l.r.nextChar) { l.r.ReadByte() // consume starting quote diff --git a/embedded/sql/parser_test.go b/embedded/sql/parser_test.go index 79ee7df449..70841503b4 100644 --- a/embedded/sql/parser_test.go +++ b/embedded/sql/parser_test.go @@ -261,7 +261,7 @@ func TestCreateTableStmt(t *testing.T) { expectedError: nil, }, { - input: "CREATE TABLE table1 (id INTEGER, name VARCHAR(50), ts TIMESTAMP, active BOOLEAN, content BLOB, PRIMARY KEY (id, name))", + input: "CREATE TABLE table1 (id INTEGER, name VARCHAR(50), ts TIMESTAMP, active BOOLEAN, content BLOB, json_data JSON, PRIMARY KEY (id, name))", expectedOutput: []SQLStmt{ &CreateTableStmt{ table: "table1", @@ -272,6 +272,7 @@ func TestCreateTableStmt(t *testing.T) { {colName: "ts", colType: TimestampType}, {colName: "active", colType: BooleanType}, {colName: "content", colType: BLOBType}, + {colName: "json_data", colType: JSONType}, }, pkColNames: []string{"id", "name"}, }}, @@ -1151,27 +1152,19 @@ func TestSelectStmt(t *testing.T) { expectedError: nil, }, { - input: "SELECT COUNT(*), SUM(age) FROM table1 ORDER BY SUM(age)", + input: "SELECT json_data->'info'->'address'->'street' FROM table1", expectedOutput: []SQLStmt{ &SelectStmt{ distinct: false, selectors: []Selector{ - &AggColSelector{aggFn: "COUNT", col: "*"}, - &AggColSelector{aggFn: "SUM", col: "age"}, - }, - ds: &tableRef{table: "table1"}, - where: nil, - orderBy: []*OrdCol{ - { - sel: &AggColSelector{ - aggFn: "SUM", - col: "age", - as: "", - }, + &JSONSelector{ + ColSelector: &ColSelector{col: "json_data"}, + fields: []string{"info", "address", "street"}, }, }, - }, - }, + ds: &tableRef{table: "table1"}, + }}, + expectedError: nil, }, } diff --git a/embedded/sql/proj_row_reader.go b/embedded/sql/proj_row_reader.go index d2642241e3..c56fa24ffe 100644 --- a/embedded/sql/proj_row_reader.go +++ b/embedded/sql/proj_row_reader.go @@ -22,8 +22,7 @@ import ( ) type projectedRowReader struct { - rowReader RowReader - + rowReader RowReader tableAlias string selectors []Selector @@ -125,15 +124,14 @@ func (pr *projectedRowReader) colsBySelector(ctx context.Context) (map[string]Co } colDescriptors := make(map[string]ColDescriptor, len(pr.selectors)) + emptyParams := make(map[string]string) for i, sel := range pr.selectors { aggFn, table, col := sel.resolve(pr.rowReader.TableAlias()) - encSel := EncodeSelector(aggFn, table, col) - - colDesc, ok := dsColDescriptors[encSel] - if !ok { - return nil, fmt.Errorf("%w (%s)", ErrColumnDoesNotExist, col) + sqlType, err := sel.inferType(dsColDescriptors, emptyParams, pr.rowReader.TableAlias()) + if err != nil { + return nil, err } if pr.tableAlias != "" { @@ -156,7 +154,7 @@ func (pr *projectedRowReader) colsBySelector(ctx context.Context) (map[string]Co AggFn: aggFn, Table: table, Column: col, - Type: colDesc.Type, + Type: sqlType, } colDescriptors[des.Selector()] = des @@ -185,15 +183,12 @@ func (pr *projectedRowReader) Read(ctx context.Context) (*Row, error) { } for i, sel := range pr.selectors { - aggFn, table, col := sel.resolve(pr.rowReader.TableAlias()) - - encSel := EncodeSelector(aggFn, table, col) - - val, ok := row.ValuesBySelector[encSel] - if !ok { - return nil, fmt.Errorf("%w (%s)", ErrColumnDoesNotExist, col) + v, err := sel.reduce(pr.Tx(), row, pr.rowReader.TableAlias()) + if err != nil { + return nil, err } + aggFn, table, col := sel.resolve(pr.rowReader.TableAlias()) if pr.tableAlias != "" { table = pr.tableAlias } @@ -210,10 +205,9 @@ func (pr *projectedRowReader) Read(ctx context.Context) (*Row, error) { } } - prow.ValuesByPosition[i] = val - prow.ValuesBySelector[EncodeSelector(aggFn, table, col)] = val + prow.ValuesByPosition[i] = v + prow.ValuesBySelector[EncodeSelector(aggFn, table, col)] = v } - return prow, nil } diff --git a/embedded/sql/row_reader.go b/embedded/sql/row_reader.go index c2d8508294..46e122481c 100644 --- a/embedded/sql/row_reader.go +++ b/embedded/sql/row_reader.go @@ -21,6 +21,7 @@ import ( "crypto/sha256" "encoding/binary" "errors" + "fmt" "math" "github.com/codenotary/immudb/embedded/store" @@ -44,11 +45,24 @@ type ScanSpecs struct { Index *Index rangesByColID map[uint32]*typedValueRange IncludeHistory bool + IncludeTxMetadata bool DescOrder bool groupBySortColumns []*OrdCol orderBySortCols []*OrdCol } +func (s *ScanSpecs) extraCols() int { + n := 0 + if s.IncludeHistory { + n++ + } + + if s.IncludeTxMetadata { + n++ + } + return n +} + type Row struct { ValuesByPosition []TypedValue ValuesBySelector map[string]TypedValue @@ -191,28 +205,34 @@ func newRawRowReader(tx *SQLTx, params map[string]interface{}, table *Table, per tableAlias = table.name } - var colsByPos []ColDescriptor - var colsBySel map[string]ColDescriptor + nCols := len(table.cols) + scanSpecs.extraCols() - var off int + colsByPos := make([]ColDescriptor, nCols) + colsBySel := make(map[string]ColDescriptor, nCols) + off := 0 if scanSpecs.IncludeHistory { - colsByPos = make([]ColDescriptor, 1+len(table.cols)) - colsBySel = make(map[string]ColDescriptor, 1+len(table.cols)) - colDescriptor := ColDescriptor{ Table: tableAlias, Column: revCol, Type: IntegerType, } - colsByPos[0] = colDescriptor + colsByPos[off] = colDescriptor colsBySel[colDescriptor.Selector()] = colDescriptor + off++ + } - off = 1 - } else { - colsByPos = make([]ColDescriptor, len(table.cols)) - colsBySel = make(map[string]ColDescriptor, len(table.cols)) + if scanSpecs.IncludeTxMetadata { + colDescriptor := ColDescriptor{ + Table: tableAlias, + Column: txMetadataCol, + Type: JSONType, + } + + colsByPos[off] = colDescriptor + colsBySel[colDescriptor.Selector()] = colDescriptor + off++ } for i, c := range table.cols { @@ -447,9 +467,15 @@ func (r *rawRowReader) Read(ctx context.Context) (*Row, error) { for i, col := range r.colsByPos { var val TypedValue - if col.Column == revCol { + switch col.Column { + case revCol: val = &Integer{val: int64(vref.HC())} - } else { + case txMetadataCol: + val, err = r.parseTxMetadata(vref.TxMetadata()) + if err != nil { + return nil, err + } + default: val = &NullValue{t: col.Type} } @@ -461,6 +487,8 @@ func (r *rawRowReader) Read(ctx context.Context) (*Row, error) { return nil, ErrCorruptedData } + extraCols := r.scanSpecs.extraCols() + voff := 0 cols := int(binary.BigEndian.Uint32(v[voff:])) @@ -505,11 +533,8 @@ func (r *rawRowReader) Read(ctx context.Context) (*Row, error) { return nil, ErrCorruptedData } - if r.scanSpecs.IncludeHistory { - valuesByPosition[pos+1] = val - } else { - valuesByPosition[pos] = val - } + valuesByPosition[pos+extraCols] = val + pos++ valuesBySelector[EncodeSelector("", r.tableAlias, col.colName)] = val @@ -522,6 +547,25 @@ func (r *rawRowReader) Read(ctx context.Context) (*Row, error) { return &Row{ValuesByPosition: valuesByPosition, ValuesBySelector: valuesBySelector}, nil } +func (r *rawRowReader) parseTxMetadata(txmd *store.TxMetadata) (TypedValue, error) { + if txmd == nil { + return &NullValue{t: JSONType}, nil + } + + if extra := txmd.Extra(); extra != nil { + if r.tx.engine.parseTxMetadata == nil { + return nil, fmt.Errorf("unable to parse tx metadata") + } + + md, err := r.tx.engine.parseTxMetadata(extra) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidTxMetadata, err) + } + return &JSON{val: md}, nil + } + return &NullValue{t: JSONType}, nil +} + func (r *rawRowReader) Close() error { if r.onCloseCallback != nil { defer r.onCloseCallback() diff --git a/embedded/sql/sort_reader.go b/embedded/sql/sort_reader.go index 1822e84328..23ea509128 100644 --- a/embedded/sql/sort_reader.go +++ b/embedded/sql/sort_reader.go @@ -31,7 +31,6 @@ type sortRowReader struct { rowReader RowReader ordCols []*OrdCol orderByDescriptors []ColDescriptor - sortKeysPositions []int sorter fileSorter resultReader resultReader @@ -57,7 +56,7 @@ func newSortRowReader(rowReader RowReader, ordCols []*OrdCol) (*sortRowReader, e return nil, err } - sortKeysPositions, err := getSortKeysPositions(colPosBySelector, ordCols, rowReader.TableAlias()) + orderByDescriptors, err := getOrderByDescriptors(ordCols, rowReader) if err != nil { return nil, err } @@ -65,9 +64,8 @@ func newSortRowReader(rowReader RowReader, ordCols []*OrdCol) (*sortRowReader, e tx := rowReader.Tx() sr := &sortRowReader{ rowReader: rowReader, - orderByDescriptors: getOrderByDescriptors(descriptors, sortKeysPositions), ordCols: ordCols, - sortKeysPositions: sortKeysPositions, + orderByDescriptors: orderByDescriptors, sorter: fileSorter{ colPosBySelector: colPosBySelector, colTypes: colTypes, @@ -85,22 +83,67 @@ func newSortRowReader(rowReader RowReader, ordCols []*OrdCol) (*sortRowReader, e } } - k1 := make(Tuple, len(sortKeysPositions)) - k2 := make(Tuple, len(sortKeysPositions)) + t1 := make(Tuple, len(ordCols)) + t2 := make(Tuple, len(ordCols)) - sr.sorter.cmp = func(t1, t2 Tuple) int { - sr.extractSortKey(t1, k1) - sr.extractSortKey(t2, k2) + sr.sorter.cmp = func(r1, r2 *Row) (int, error) { + if err := sr.evalSortSelectors(r1, t1); err != nil { + return 0, err + } + + if err := sr.evalSortSelectors(r2, t2); err != nil { + return 0, err + } + + res, idx, err := t1.Compare(t2) + if err != nil { + return 0, err + } - res, idx, _ := k1.Compare(k2) if idx >= 0 { - return res * int(directions[idx]) + return res * int(directions[idx]), nil } - return res + return res, nil } return sr, nil } +func (s *sortRowReader) evalSortSelectors(inRow *Row, out Tuple) error { + for i, col := range s.ordCols { + val, err := col.sel.reduce(s.Tx(), inRow, s.TableAlias()) + if err != nil { + return err + } + out[i] = val + } + return nil +} + +func getOrderByDescriptors(ordCols []*OrdCol, rowReader RowReader) ([]ColDescriptor, error) { + colsBySel, err := rowReader.colsBySelector(context.Background()) + if err != nil { + return nil, err + } + + params := make(map[string]string) + orderByDescriptors := make([]ColDescriptor, len(ordCols)) + for i, col := range ordCols { + sqlType, err := col.sel.inferType(colsBySel, params, rowReader.TableAlias()) + if err != nil { + return nil, err + } + + aggFn, table, col := col.sel.resolve(rowReader.TableAlias()) + orderByDescriptors[i] = ColDescriptor{ + AggFn: aggFn, + Table: table, + Column: col, + Type: sqlType, + } + } + return orderByDescriptors, nil +} + func getColTypes(r RowReader) ([]string, error) { descriptors, err := r.Columns(context.Background()) if err != nil { @@ -114,21 +157,6 @@ func getColTypes(r RowReader) ([]string, error) { return cols, err } -func getSortKeysPositions(colPosBySelector map[string]int, cols []*OrdCol, tableAlias string) ([]int, error) { - sortKeysPositions := make([]int, len(cols)) - for i, col := range cols { - aggFn, table, col := col.sel.resolve(tableAlias) - encSel := EncodeSelector(aggFn, table, col) - - pos, exists := colPosBySelector[encSel] - if !exists { - return nil, ErrColumnDoesNotExist - } - sortKeysPositions[i] = pos - } - return sortKeysPositions, nil -} - func getColPositionsBySelector(desc []ColDescriptor) (map[string]int, error) { colPositionsBySelector := make(map[string]int) for i, desc := range desc { @@ -137,20 +165,6 @@ func getColPositionsBySelector(desc []ColDescriptor) (map[string]int, error) { return colPositionsBySelector, nil } -func (sr *sortRowReader) extractSortKey(t Tuple, out Tuple) { - for i, pos := range sr.sortKeysPositions { - out[i] = t[pos] - } -} - -func getOrderByDescriptors(descriptors []ColDescriptor, sortKeysPositions []int) []ColDescriptor { - orderByDescriptors := make([]ColDescriptor, len(sortKeysPositions)) - for i, pos := range sortKeysPositions { - orderByDescriptors[i] = descriptors[pos] - } - return orderByDescriptors -} - func (sr *sortRowReader) onClose(callback func()) { sr.rowReader.onClose(callback) } diff --git a/embedded/sql/sql_grammar.y b/embedded/sql/sql_grammar.y index 0d1767d939..88f24aa92a 100644 --- a/embedded/sql/sql_grammar.y +++ b/embedded/sql/sql_grammar.y @@ -47,6 +47,7 @@ func setResult(l yyLexer, stmts []SQLStmt) { col *ColSelector sel Selector sels []Selector + jsonFields []string distinct bool ds DataSource tableRef *tableRef @@ -93,6 +94,7 @@ func setResult(l yyLexer, stmts []SQLStmt) { %token AGGREGATE_FUNC %token ERROR %token DOT +%token ARROW %left ',' %right AS @@ -118,6 +120,7 @@ func setResult(l yyLexer, stmts []SQLStmt) { %type val fnCall %type selector %type opt_selectors selectors +%type jsonFields %type col %type opt_distinct opt_all %type ds @@ -657,6 +660,11 @@ selector: { $$ = $1 } +| + col jsonFields + { + $$ = &JSONSelector{ColSelector: $1, fields: $2} + } | AGGREGATE_FUNC '(' '*' ')' { @@ -668,6 +676,17 @@ selector: $$ = &AggColSelector{aggFn: $1, table: $3.table, col: $3.col} } +jsonFields: + ARROW VARCHAR + { + $$ = []string{$2} + } +| + jsonFields ARROW VARCHAR + { + $$ = append($$, $3) + } + col: IDENTIFIER { diff --git a/embedded/sql/sql_parser.go b/embedded/sql/sql_parser.go index c65c81e00f..ee081ad5ad 100644 --- a/embedded/sql/sql_parser.go +++ b/embedded/sql/sql_parser.go @@ -33,6 +33,7 @@ type yySymType struct { col *ColSelector sel Selector sels []Selector + jsonFields []string distinct bool ds DataSource tableRef *tableRef @@ -145,7 +146,8 @@ const BLOB = 57431 const AGGREGATE_FUNC = 57432 const ERROR = 57433 const DOT = 57434 -const STMT_SEPARATOR = 57435 +const ARROW = 57435 +const STMT_SEPARATOR = 57436 var yyToknames = [...]string{ "$end", @@ -240,6 +242,7 @@ var yyToknames = [...]string{ "AGGREGATE_FUNC", "ERROR", "DOT", + "ARROW", "','", "'+'", "'-'", @@ -264,137 +267,138 @@ var yyExca = [...]int16{ 1, -1, -2, 0, -1, 93, - 65, 161, - 68, 161, - -2, 149, - -1, 227, - 51, 125, - -2, 120, - -1, 267, - 51, 125, - -2, 122, + 65, 164, + 68, 164, + -2, 152, + -1, 231, + 51, 128, + -2, 123, + -1, 272, + 51, 128, + -2, 125, } const yyPrivate = 57344 -const yyLast = 447 +const yyLast = 452 var yyAct = [...]int16{ - 92, 359, 98, 78, 260, 129, 221, 169, 290, 294, - 107, 175, 166, 209, 289, 266, 210, 6, 121, 243, - 190, 57, 124, 20, 330, 180, 281, 219, 280, 253, - 219, 341, 219, 219, 334, 335, 219, 315, 313, 95, - 282, 254, 97, 331, 220, 325, 110, 106, 316, 19, - 77, 314, 295, 108, 109, 148, 305, 276, 111, 91, - 101, 102, 103, 104, 105, 79, 274, 146, 147, 296, - 96, 273, 271, 252, 250, 100, 291, 178, 179, 181, - 142, 143, 145, 144, 238, 183, 237, 207, 148, 126, - 218, 133, 141, 249, 242, 157, 152, 153, 234, 157, - 233, 155, 177, 95, 232, 231, 97, 192, 158, 156, - 110, 106, 154, 142, 143, 145, 144, 108, 109, 358, - 135, 132, 111, 171, 101, 102, 103, 104, 105, 79, - 120, 184, 251, 168, 96, 119, 187, 22, 182, 100, - 148, 80, 172, 195, 196, 197, 198, 199, 200, 80, - 352, 148, 146, 147, 186, 319, 79, 208, 211, 148, - 122, 318, 75, 146, 147, 142, 143, 145, 144, 253, - 239, 212, 206, 219, 128, 226, 142, 143, 145, 144, - 224, 213, 90, 227, 133, 114, 145, 144, 235, 217, - 236, 194, 80, 312, 95, 229, 225, 97, 228, 79, - 173, 110, 106, 311, 248, 241, 286, 131, 108, 109, - 277, 240, 205, 111, 148, 101, 102, 103, 104, 105, - 79, 62, 318, 262, 80, 96, 167, 147, 30, 130, - 100, 264, 302, 288, 270, 31, 275, 256, 258, 142, - 143, 145, 144, 112, 125, 216, 215, 214, 191, 193, - 211, 188, 185, 159, 287, 283, 136, 113, 83, 81, - 278, 191, 293, 10, 12, 11, 285, 284, 73, 297, - 41, 139, 140, 66, 292, 65, 64, 301, 63, 303, - 304, 61, 306, 299, 298, 56, 13, 55, 174, 269, - 20, 45, 211, 7, 329, 8, 9, 14, 15, 151, - 247, 16, 17, 230, 29, 320, 328, 20, 150, 321, - 148, 182, 324, 310, 202, 134, 19, 326, 51, 203, - 309, 201, 204, 42, 82, 72, 50, 332, 360, 361, - 344, 340, 339, 19, 261, 222, 20, 351, 345, 338, - 323, 337, 347, 43, 44, 46, 122, 300, 127, 353, - 350, 39, 52, 53, 356, 354, 357, 48, 349, 342, - 333, 362, 19, 70, 363, 176, 259, 257, 38, 37, - 23, 307, 161, 162, 163, 160, 117, 255, 348, 24, - 28, 85, 263, 40, 137, 84, 223, 54, 2, 272, - 164, 34, 36, 89, 88, 25, 27, 26, 138, 115, - 116, 59, 60, 67, 68, 69, 32, 35, 33, 118, - 170, 49, 244, 245, 246, 86, 21, 317, 123, 149, - 308, 327, 343, 355, 279, 322, 94, 93, 336, 268, - 267, 265, 87, 58, 71, 47, 76, 74, 99, 346, - 165, 189, 18, 5, 4, 3, 1, + 92, 364, 98, 78, 265, 129, 225, 171, 295, 299, + 177, 107, 168, 213, 294, 271, 214, 6, 121, 248, + 194, 57, 124, 20, 335, 286, 182, 285, 223, 258, + 223, 346, 223, 223, 339, 223, 340, 320, 318, 95, + 287, 259, 97, 224, 336, 330, 110, 106, 300, 19, + 77, 91, 321, 108, 109, 150, 319, 310, 111, 281, + 101, 102, 103, 104, 105, 79, 301, 148, 149, 279, + 278, 96, 276, 257, 255, 243, 100, 296, 180, 181, + 183, 144, 145, 147, 146, 242, 185, 222, 211, 126, + 135, 254, 143, 247, 159, 95, 154, 155, 97, 159, + 238, 157, 110, 106, 179, 237, 236, 235, 196, 108, + 109, 160, 158, 156, 111, 137, 101, 102, 103, 104, + 105, 79, 134, 173, 120, 119, 22, 96, 363, 150, + 80, 186, 100, 170, 324, 256, 357, 79, 191, 184, + 80, 188, 174, 150, 75, 199, 200, 201, 202, 203, + 204, 133, 150, 122, 190, 148, 149, 147, 146, 212, + 215, 323, 258, 244, 148, 149, 223, 128, 135, 144, + 145, 147, 146, 216, 210, 241, 114, 230, 144, 145, + 147, 146, 228, 217, 80, 231, 90, 80, 317, 221, + 239, 79, 240, 198, 175, 316, 95, 233, 229, 97, + 232, 189, 323, 110, 106, 30, 291, 282, 253, 246, + 108, 109, 31, 245, 131, 111, 150, 101, 102, 103, + 104, 105, 79, 150, 209, 169, 62, 267, 96, 149, + 274, 307, 293, 100, 280, 269, 130, 176, 275, 263, + 125, 261, 144, 145, 147, 146, 220, 112, 219, 144, + 145, 147, 146, 218, 195, 215, 197, 192, 195, 292, + 288, 187, 161, 138, 113, 283, 83, 298, 10, 12, + 11, 290, 289, 81, 302, 141, 142, 73, 41, 297, + 66, 29, 306, 63, 308, 309, 65, 311, 304, 303, + 64, 13, 61, 56, 55, 20, 45, 215, 7, 334, + 8, 9, 14, 15, 153, 252, 16, 17, 234, 333, + 325, 150, 20, 152, 326, 136, 329, 184, 315, 206, + 82, 19, 331, 51, 207, 314, 205, 208, 72, 42, + 365, 366, 337, 349, 266, 226, 345, 344, 19, 356, + 343, 20, 328, 350, 50, 122, 342, 352, 43, 44, + 46, 305, 127, 39, 358, 355, 48, 354, 347, 361, + 359, 362, 338, 70, 178, 264, 367, 19, 262, 368, + 52, 53, 38, 37, 23, 312, 163, 164, 165, 162, + 260, 34, 40, 117, 353, 268, 139, 84, 227, 54, + 36, 2, 24, 28, 172, 277, 32, 166, 33, 85, + 59, 60, 67, 68, 69, 35, 115, 116, 25, 27, + 26, 89, 88, 140, 49, 249, 250, 251, 118, 86, + 21, 322, 123, 151, 313, 332, 348, 360, 284, 327, + 94, 93, 341, 273, 272, 270, 87, 58, 71, 47, + 132, 76, 74, 99, 351, 167, 193, 18, 5, 4, + 3, 1, } var yyPact = [...]int16{ - 259, -1000, -1000, 38, -1000, -1000, -1000, 335, -1000, -1000, - 372, 221, 383, 384, 329, 328, 301, 187, 261, 268, - 308, -1000, 259, -1000, 252, 252, 252, 362, 204, -1000, - 202, 385, 198, 195, 193, 192, 190, 187, 187, 187, - 319, -1000, 262, -1000, -1000, 185, -1000, 66, -1000, -1000, - 176, 260, 175, 359, 252, 406, -1000, -1000, 375, 39, - 39, -1000, 174, 93, -1000, 371, 400, 35, 30, 293, - 161, 242, -1000, -1000, 298, -1000, 81, 146, -1000, 21, - 92, -1000, 248, 20, 173, 358, 388, -1000, 39, 39, - -1000, 130, 82, 235, -1000, 130, 130, 12, -1000, -1000, - 130, -1000, -1000, -1000, -1000, -1000, 9, -1000, -1000, -1000, - -1000, -1, -1000, 8, 170, 344, 342, 343, 380, 143, - 143, 404, 130, 107, -1000, 206, -1000, 2, 109, -1000, - -1000, 169, 58, 168, -1000, 165, 7, 166, 104, -1000, - -1000, 82, 130, 130, 130, 130, 130, 130, 250, 254, - 128, -1000, 145, 90, 242, -14, 130, 130, 143, -1000, - 165, 164, 163, 162, 102, -11, 80, -1000, -57, 279, - 361, 82, 404, 161, 130, 404, 385, 288, 5, 4, - 0, -2, 146, -5, 146, -1000, -15, -17, -1000, 77, - -1000, 127, 143, -6, 401, 90, 90, 241, 241, 145, - 19, -1000, 229, 130, -7, -1000, -27, -1000, 71, -28, - 76, 82, -60, -1000, -1000, 347, -1000, 401, 326, 155, - 325, 277, 130, 356, 279, -1000, 82, 209, 146, -29, - 368, -30, -35, 153, -44, -1000, -1000, -1000, -1000, 178, - -74, -61, 143, -1000, -1000, -1000, -1000, -1000, 145, -25, - -1000, 122, -1000, 130, -1000, 150, -1000, -24, -1000, -24, - -1000, 130, 82, -31, 277, 293, -1000, 209, 296, -1000, - -1000, 146, 149, 146, 146, -45, 146, 338, -1000, 249, - 118, 108, -1000, -63, -50, -64, -53, 82, -1000, 129, - -1000, 130, 68, 82, -1000, -1000, 143, -1000, 286, -1000, - 2, -1000, -56, -1000, -1000, -1000, -1000, -31, 236, -1000, - 223, -79, -58, -1000, -1000, -1000, -1000, -1000, -24, 315, - -67, -66, 289, 284, 404, 146, -70, -1000, -1000, -1000, - -1000, -1000, -1000, 313, -1000, -1000, 272, 130, 141, 352, - -1000, -1000, 311, 279, 282, 82, 57, -1000, 130, -1000, - 277, 109, 141, 82, -1000, 26, 269, -1000, 109, -1000, - -1000, -1000, 269, -1000, + 264, -1000, -1000, 26, -1000, -1000, -1000, 339, -1000, -1000, + 385, 198, 373, 382, 333, 332, 303, 195, 267, 273, + 307, -1000, 264, -1000, 257, 257, 257, 364, 211, -1000, + 210, 384, 209, 200, 207, 203, 197, 195, 195, 195, + 319, -1000, 265, -1000, -1000, 194, -1000, 47, -1000, -1000, + 190, 256, 183, 361, 257, 410, -1000, -1000, 393, 31, + 31, -1000, 181, 84, -1000, 378, 409, 24, 23, 292, + 157, 247, -1000, -1000, 302, -1000, 73, 153, 58, 21, + 76, -1000, 248, 14, 180, 360, 403, -1000, 31, 31, + -1000, 132, 83, 240, -1000, 132, 132, 12, -1000, -1000, + 132, -1000, -1000, -1000, -1000, -1000, 11, -1000, -1000, -1000, + -1000, -2, -1000, 10, 179, 348, 346, 347, 387, 142, + 142, 388, 132, 100, -1000, 155, -1000, 3, 101, -1000, + -1000, 178, 48, 114, 57, 174, -1000, 171, 7, 173, + 106, -1000, -1000, 83, 132, 132, 132, 132, 132, 132, + 255, 259, 140, -1000, 147, 60, 247, -14, 132, 132, + 142, -1000, 171, 170, 165, 163, 102, -15, 72, -1000, + -59, 279, 363, 83, 388, 157, 132, 388, 384, 293, + 6, 5, 4, -1, 153, -7, 153, -1000, 88, -1000, + -17, -27, -1000, 69, -1000, 129, 142, -8, 404, 60, + 60, 242, 242, 147, 154, -1000, 234, 132, -10, -1000, + -28, -1000, 74, -29, 68, 83, -61, -1000, -1000, 350, + -1000, 404, 327, 156, 324, 277, 132, 359, 279, -1000, + 83, 150, 153, -30, 374, -32, -33, 151, -43, -1000, + -1000, -1000, -1000, -1000, 175, -76, -62, 142, -1000, -1000, + -1000, -1000, -1000, 147, -25, -1000, 122, -1000, 132, -1000, + 149, -1000, -24, -1000, -24, -1000, 132, 83, -35, 277, + 292, -1000, 150, 300, -1000, -1000, 153, 148, 153, 153, + -45, 153, 342, -1000, 254, 110, 103, -1000, -64, -46, + -65, -50, 83, -1000, 108, -1000, 132, 67, 83, -1000, + -1000, 142, -1000, 288, -1000, 3, -1000, -57, -1000, -1000, + -1000, -1000, -35, 239, -1000, 228, -80, -58, -1000, -1000, + -1000, -1000, -1000, -24, 317, -68, -66, 294, 285, 388, + 153, -71, -1000, -1000, -1000, -1000, -1000, -1000, 312, -1000, + -1000, 275, 132, 104, 358, -1000, -1000, 310, 279, 284, + 83, 42, -1000, 132, -1000, 277, 101, 104, 83, -1000, + 34, 271, -1000, 101, -1000, -1000, -1000, 271, -1000, } var yyPgo = [...]int16{ - 0, 446, 388, 445, 444, 443, 17, 442, 441, 20, - 12, 9, 440, 439, 14, 8, 16, 13, 438, 10, - 2, 437, 436, 3, 435, 434, 11, 365, 21, 433, - 432, 182, 431, 15, 430, 429, 0, 18, 428, 427, - 426, 425, 6, 4, 424, 5, 423, 422, 1, 7, - 326, 421, 420, 419, 22, 418, 417, 19, 416, + 0, 451, 391, 450, 449, 448, 17, 447, 446, 20, + 12, 9, 445, 444, 14, 8, 16, 13, 443, 11, + 2, 442, 441, 440, 3, 439, 438, 10, 364, 21, + 437, 436, 186, 435, 15, 434, 433, 0, 18, 432, + 431, 430, 429, 6, 4, 428, 5, 427, 426, 1, + 7, 344, 425, 424, 423, 22, 422, 421, 19, 420, } var yyR1 = [...]int8{ - 0, 1, 2, 2, 58, 58, 3, 3, 3, 4, + 0, 1, 2, 2, 59, 59, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 57, 57, 57, 57, 50, 50, 11, 11, 5, 5, - 5, 5, 56, 56, 55, 55, 54, 12, 12, 14, + 58, 58, 58, 58, 51, 51, 11, 11, 5, 5, + 5, 5, 57, 57, 56, 56, 55, 12, 12, 14, 14, 15, 10, 10, 13, 13, 17, 17, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 19, 8, 8, 9, 44, 44, 44, 51, 51, 52, - 52, 52, 6, 6, 6, 6, 6, 6, 7, 25, - 25, 24, 24, 21, 21, 22, 22, 20, 20, 20, - 23, 23, 26, 26, 26, 26, 26, 26, 26, 26, - 27, 28, 29, 29, 29, 30, 30, 30, 31, 31, - 32, 32, 33, 33, 34, 35, 35, 37, 37, 41, - 41, 38, 38, 42, 42, 43, 43, 47, 47, 49, - 49, 46, 46, 48, 48, 48, 45, 45, 45, 36, - 36, 36, 36, 36, 36, 36, 36, 39, 39, 39, - 39, 53, 53, 40, 40, 40, 40, 40, 40, 40, - 40, + 19, 8, 8, 9, 45, 45, 45, 52, 52, 53, + 53, 53, 6, 6, 6, 6, 6, 6, 7, 26, + 26, 25, 25, 21, 21, 22, 22, 20, 20, 20, + 20, 23, 23, 24, 24, 27, 27, 27, 27, 27, + 27, 27, 27, 28, 29, 30, 30, 30, 31, 31, + 31, 32, 32, 33, 33, 34, 34, 35, 36, 36, + 38, 38, 42, 42, 39, 39, 43, 43, 44, 44, + 48, 48, 50, 50, 47, 47, 49, 49, 49, 46, + 46, 46, 37, 37, 37, 37, 37, 37, 37, 37, + 40, 40, 40, 40, 54, 54, 41, 41, 41, 41, + 41, 41, 41, 41, } var yyR2 = [...]int8{ @@ -407,95 +411,95 @@ var yyR2 = [...]int8{ 1, 1, 1, 1, 1, 6, 1, 1, 1, 1, 4, 1, 3, 5, 0, 3, 3, 0, 1, 0, 1, 2, 1, 4, 2, 2, 3, 2, 13, 0, - 1, 0, 1, 1, 1, 2, 4, 1, 4, 4, - 1, 3, 3, 4, 4, 4, 4, 4, 2, 6, - 1, 2, 0, 2, 2, 0, 2, 2, 2, 1, - 0, 1, 1, 2, 6, 0, 1, 0, 2, 0, - 3, 0, 2, 0, 2, 0, 2, 0, 3, 0, - 4, 2, 4, 0, 1, 1, 0, 1, 2, 1, - 1, 2, 2, 4, 4, 6, 6, 1, 1, 3, - 3, 0, 1, 3, 3, 3, 3, 3, 3, 3, - 4, + 1, 0, 1, 1, 1, 2, 4, 1, 2, 4, + 4, 2, 3, 1, 3, 3, 4, 4, 4, 4, + 4, 2, 6, 1, 2, 0, 2, 2, 0, 2, + 2, 2, 1, 0, 1, 1, 2, 6, 0, 1, + 0, 2, 0, 3, 0, 2, 0, 2, 0, 2, + 0, 3, 0, 4, 2, 4, 0, 1, 1, 0, + 1, 2, 1, 1, 2, 2, 4, 4, 6, 6, + 1, 1, 3, 3, 0, 1, 3, 3, 3, 3, + 3, 3, 3, 4, } var yyChk = [...]int16{ -1000, -1, -2, -3, -4, -5, -6, 34, 36, 37, 4, 6, 5, 27, 38, 39, 42, 43, -7, 74, - 48, -58, 99, 35, 7, 23, 25, 24, 8, 83, + 48, -59, 100, 35, 7, 23, 25, 24, 8, 83, 7, 14, 23, 25, 8, 23, 8, 40, 40, 50, - -27, 83, 62, 75, 76, 23, 77, -24, 49, -2, - -50, 66, -50, -50, 25, 83, 83, -28, -29, 16, - 17, 83, 26, 83, 83, 83, 83, -27, -27, -27, - 44, -25, 63, 83, -21, 96, -22, -20, -23, 90, - 83, 83, 64, 83, 26, -50, 9, -30, 19, 18, - -31, 20, -36, -39, -40, 64, 95, 67, -20, -18, - 100, 85, 86, 87, 88, 89, 72, -19, 78, 79, - 71, 83, -31, 83, 92, 28, 29, 5, 9, 100, - 100, -37, 53, -55, -54, 83, -6, 50, 93, -45, - 83, 61, 100, 92, 67, 100, 83, 26, 10, -31, - -31, -36, 94, 95, 97, 96, 81, 82, 69, -53, - 73, 64, -36, -36, 100, -36, 100, 100, 100, 83, - 31, 30, 31, 31, 10, -12, -10, 83, -10, -49, - 6, -36, -37, 93, 82, -26, -27, 100, 75, 76, - 23, 77, -19, 83, -20, 83, 96, -23, 83, -8, - -9, 83, 100, 83, 87, -36, -36, -36, -36, -36, - -36, 71, 64, 65, 68, 84, -6, 101, -36, -17, - -16, -36, -10, -9, 83, 83, 83, 87, 101, 93, - 101, -42, 56, 25, -49, -54, -36, -49, -28, -6, - 15, 100, 100, 100, 100, -45, -45, 101, 101, 93, - 84, -10, 100, -57, 11, 12, 13, 71, -36, 100, - 101, 61, 101, 93, 101, 30, -57, 41, 83, 41, - -43, 57, -36, 26, -42, -32, -33, -34, -35, 80, - -45, 101, 21, 101, 101, 83, 101, 32, -9, -44, - 102, 100, 101, -10, -6, -16, 84, -36, 83, -14, - -15, 100, -14, -36, -11, 83, 100, -43, -37, -33, - 51, -45, 83, -45, -45, 101, -45, 33, -52, 71, - 64, 85, 85, 101, 101, 101, 101, -56, 93, 26, - -17, -10, -41, 54, -26, 101, -11, -51, 70, 71, - 103, 101, -15, 45, 101, 101, -38, 52, 55, -49, - -45, 101, 46, -47, 58, -36, -13, -23, 26, 47, - -42, 55, 93, -36, -43, -46, -20, -23, 93, -48, - 59, 60, -20, -48, + -28, 83, 62, 75, 76, 23, 77, -25, 49, -2, + -51, 66, -51, -51, 25, 83, 83, -29, -30, 16, + 17, 83, 26, 83, 83, 83, 83, -28, -28, -28, + 44, -26, 63, 83, -21, 97, -22, -20, -24, 90, + 83, 83, 64, 83, 26, -51, 9, -31, 19, 18, + -32, 20, -37, -40, -41, 64, 96, 67, -20, -18, + 101, 85, 86, 87, 88, 89, 72, -19, 78, 79, + 71, 83, -32, 83, 92, 28, 29, 5, 9, 101, + 101, -38, 53, -56, -55, 83, -6, 50, 94, -46, + 83, 61, -23, 93, 101, 92, 67, 101, 83, 26, + 10, -32, -32, -37, 95, 96, 98, 97, 81, 82, + 69, -54, 73, 64, -37, -37, 101, -37, 101, 101, + 101, 83, 31, 30, 31, 31, 10, -12, -10, 83, + -10, -50, 6, -37, -38, 94, 82, -27, -28, 101, + 75, 76, 23, 77, -19, 83, -20, 83, 93, 87, + 97, -24, 83, -8, -9, 83, 101, 83, 87, -37, + -37, -37, -37, -37, -37, 71, 64, 65, 68, 84, + -6, 102, -37, -17, -16, -37, -10, -9, 83, 83, + 83, 87, 102, 94, 102, -43, 56, 25, -50, -55, + -37, -50, -29, -6, 15, 101, 101, 101, 101, -46, + -46, 87, 102, 102, 94, 84, -10, 101, -58, 11, + 12, 13, 71, -37, 101, 102, 61, 102, 94, 102, + 30, -58, 41, 83, 41, -44, 57, -37, 26, -43, + -33, -34, -35, -36, 80, -46, 102, 21, 102, 102, + 83, 102, 32, -9, -45, 103, 101, 102, -10, -6, + -16, 84, -37, 83, -14, -15, 101, -14, -37, -11, + 83, 101, -44, -38, -34, 51, -46, 83, -46, -46, + 102, -46, 33, -53, 71, 64, 85, 85, 102, 102, + 102, 102, -57, 94, 26, -17, -10, -42, 54, -27, + 102, -11, -52, 70, 71, 104, 102, -15, 45, 102, + 102, -39, 52, 55, -50, -46, 102, 46, -48, 58, + -37, -13, -24, 26, 47, -43, 55, 94, -37, -44, + -47, -20, -24, 94, -49, 59, 60, -20, -49, } var yyDef = [...]int16{ 0, -2, 1, 4, 6, 7, 8, 10, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 91, 2, 5, 9, 34, 34, 34, 0, 0, 14, - 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 110, 89, 84, 85, 0, 87, 0, 92, 3, - 0, 0, 0, 0, 34, 0, 15, 16, 115, 0, - 0, 18, 0, 0, 29, 0, 0, 0, 0, 127, - 0, 0, 90, 86, 0, 93, 94, 146, 97, 0, - 100, 13, 0, 0, 0, 0, 0, 111, 0, 0, - 113, 0, 119, -2, 150, 0, 0, 0, 157, 158, + 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 113, 89, 84, 85, 0, 87, 0, 92, 3, + 0, 0, 0, 0, 34, 0, 15, 16, 118, 0, + 0, 18, 0, 0, 29, 0, 0, 0, 0, 130, + 0, 0, 90, 86, 0, 93, 94, 149, 97, 0, + 103, 13, 0, 0, 0, 0, 0, 114, 0, 0, + 116, 0, 122, -2, 153, 0, 0, 0, 160, 161, 0, 60, 61, 62, 63, 64, 0, 66, 67, 68, - 69, 100, 114, 0, 0, 0, 0, 0, 0, 47, - 0, 139, 0, 127, 44, 0, 83, 0, 0, 95, - 147, 0, 0, 0, 35, 0, 0, 0, 0, 116, - 117, 118, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 162, 151, 152, 0, 0, 0, 56, 0, 22, - 0, 0, 0, 0, 0, 0, 48, 52, 0, 133, - 0, 128, 139, 0, 0, 139, 112, 0, 0, 0, - 0, 0, 146, 110, 146, 148, 0, 0, 101, 0, - 71, 0, 0, 0, 30, 163, 164, 165, 166, 167, - 168, 169, 0, 0, 0, 160, 0, 159, 0, 0, - 57, 58, 0, 23, 24, 0, 26, 30, 0, 0, - 0, 135, 0, 0, 133, 45, 46, -2, 146, 0, - 0, 0, 0, 0, 0, 108, 96, 98, 99, 0, - 74, 0, 0, 27, 31, 32, 33, 170, 153, 0, - 154, 0, 70, 0, 21, 0, 28, 0, 53, 0, - 40, 0, 134, 0, 135, 127, 121, -2, 0, 126, - 102, 146, 0, 146, 146, 0, 146, 0, 72, 79, - 0, 0, 19, 0, 0, 0, 0, 59, 25, 42, - 49, 56, 39, 136, 140, 36, 0, 41, 129, 123, - 0, 103, 0, 104, 105, 106, 107, 0, 77, 80, - 0, 0, 0, 20, 155, 156, 65, 38, 0, 0, - 0, 0, 131, 0, 139, 146, 0, 73, 78, 81, - 75, 76, 50, 0, 51, 37, 137, 0, 0, 0, - 109, 17, 0, 133, 0, 132, 130, 54, 0, 43, - 135, 0, 0, 124, 88, 138, 143, 55, 0, 141, - 144, 145, 143, 142, + 69, 103, 117, 0, 0, 0, 0, 0, 0, 47, + 0, 142, 0, 130, 44, 0, 83, 0, 0, 95, + 150, 0, 98, 0, 0, 0, 35, 0, 0, 0, + 0, 119, 120, 121, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 165, 154, 155, 0, 0, 0, 56, + 0, 22, 0, 0, 0, 0, 0, 0, 48, 52, + 0, 136, 0, 131, 142, 0, 0, 142, 115, 0, + 0, 0, 0, 0, 149, 113, 149, 151, 0, 101, + 0, 0, 104, 0, 71, 0, 0, 0, 30, 166, + 167, 168, 169, 170, 171, 172, 0, 0, 0, 163, + 0, 162, 0, 0, 57, 58, 0, 23, 24, 0, + 26, 30, 0, 0, 0, 138, 0, 0, 136, 45, + 46, -2, 149, 0, 0, 0, 0, 0, 0, 111, + 96, 102, 99, 100, 0, 74, 0, 0, 27, 31, + 32, 33, 173, 156, 0, 157, 0, 70, 0, 21, + 0, 28, 0, 53, 0, 40, 0, 137, 0, 138, + 130, 124, -2, 0, 129, 105, 149, 0, 149, 149, + 0, 149, 0, 72, 79, 0, 0, 19, 0, 0, + 0, 0, 59, 25, 42, 49, 56, 39, 139, 143, + 36, 0, 41, 132, 126, 0, 106, 0, 107, 108, + 109, 110, 0, 77, 80, 0, 0, 0, 20, 158, + 159, 65, 38, 0, 0, 0, 0, 134, 0, 142, + 149, 0, 73, 78, 81, 75, 76, 50, 0, 51, + 37, 140, 0, 0, 0, 112, 17, 0, 136, 0, + 135, 133, 54, 0, 43, 138, 0, 0, 127, 88, + 141, 146, 55, 0, 144, 147, 148, 146, 145, } var yyTok1 = [...]int8{ @@ -503,12 +507,12 @@ var yyTok1 = [...]int8{ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 100, 101, 96, 94, 93, 95, 98, 97, 3, 3, + 101, 102, 97, 95, 94, 96, 99, 98, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 102, 3, 103, + 3, 103, 3, 104, } var yyTok2 = [...]int8{ @@ -521,7 +525,7 @@ var yyTok2 = [...]int8{ 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 99, + 92, 93, 100, } var yyTok3 = [...]int8{ @@ -1355,369 +1359,384 @@ yydefault: yyVAL.sel = yyDollar[1].col } case 98: + yyDollar = yyS[yypt-2 : yypt+1] + { + yyVAL.sel = &JSONSelector{ColSelector: yyDollar[1].col, fields: yyDollar[2].jsonFields} + } + case 99: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.sel = &AggColSelector{aggFn: yyDollar[1].aggFn, col: "*"} } - case 99: + case 100: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.sel = &AggColSelector{aggFn: yyDollar[1].aggFn, table: yyDollar[3].col.table, col: yyDollar[3].col.col} } - case 100: + case 101: + yyDollar = yyS[yypt-2 : yypt+1] + { + yyVAL.jsonFields = []string{yyDollar[2].str} + } + case 102: + yyDollar = yyS[yypt-3 : yypt+1] + { + yyVAL.jsonFields = append(yyVAL.jsonFields, yyDollar[3].str) + } + case 103: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.col = &ColSelector{col: yyDollar[1].id} } - case 101: + case 104: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.col = &ColSelector{table: yyDollar[1].id, col: yyDollar[3].id} } - case 102: + case 105: yyDollar = yyS[yypt-3 : yypt+1] { yyDollar[1].tableRef.period = yyDollar[2].period yyDollar[1].tableRef.as = yyDollar[3].id yyVAL.ds = yyDollar[1].tableRef } - case 103: + case 106: yyDollar = yyS[yypt-4 : yypt+1] { yyDollar[2].stmt.(*SelectStmt).as = yyDollar[4].id yyVAL.ds = yyDollar[2].stmt.(DataSource) } - case 104: + case 107: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.ds = &FnDataSourceStmt{fnCall: &FnCall{fn: "databases"}, as: yyDollar[4].id} } - case 105: + case 108: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.ds = &FnDataSourceStmt{fnCall: &FnCall{fn: "tables"}, as: yyDollar[4].id} } - case 106: + case 109: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.ds = &FnDataSourceStmt{fnCall: &FnCall{fn: "table", params: []ValueExp{&Varchar{val: yyDollar[3].id}}}} } - case 107: + case 110: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.ds = &FnDataSourceStmt{fnCall: &FnCall{fn: "users"}, as: yyDollar[4].id} } - case 108: + case 111: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.ds = &FnDataSourceStmt{fnCall: yyDollar[1].value.(*FnCall), as: yyDollar[2].id} } - case 109: + case 112: yyDollar = yyS[yypt-6 : yypt+1] { yyVAL.ds = &tableRef{table: yyDollar[4].id, history: true, as: yyDollar[6].id} } - case 110: + case 113: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.tableRef = &tableRef{table: yyDollar[1].id} } - case 111: + case 114: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.period = period{start: yyDollar[1].openPeriod, end: yyDollar[2].openPeriod} } - case 112: + case 115: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.openPeriod = nil } - case 113: + case 116: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.openPeriod = &openPeriod{inclusive: true, instant: yyDollar[2].periodInstant} } - case 114: + case 117: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.openPeriod = &openPeriod{instant: yyDollar[2].periodInstant} } - case 115: + case 118: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.openPeriod = nil } - case 116: + case 119: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.openPeriod = &openPeriod{inclusive: true, instant: yyDollar[2].periodInstant} } - case 117: + case 120: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.openPeriod = &openPeriod{instant: yyDollar[2].periodInstant} } - case 118: + case 121: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.periodInstant = periodInstant{instantType: txInstant, exp: yyDollar[2].exp} } - case 119: + case 122: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.periodInstant = periodInstant{instantType: timeInstant, exp: yyDollar[1].exp} } - case 120: + case 123: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.joins = nil } - case 121: + case 124: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.joins = yyDollar[1].joins } - case 122: + case 125: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.joins = []*JoinSpec{yyDollar[1].join} } - case 123: + case 126: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.joins = append([]*JoinSpec{yyDollar[1].join}, yyDollar[2].joins...) } - case 124: + case 127: yyDollar = yyS[yypt-6 : yypt+1] { yyVAL.join = &JoinSpec{joinType: yyDollar[1].joinType, ds: yyDollar[3].ds, indexOn: yyDollar[4].ids, cond: yyDollar[6].exp} } - case 125: + case 128: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.joinType = InnerJoin } - case 126: + case 129: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.joinType = yyDollar[1].joinType } - case 127: + case 130: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.exp = nil } - case 128: + case 131: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.exp = yyDollar[2].exp } - case 129: + case 132: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.cols = nil } - case 130: + case 133: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.cols = yyDollar[3].cols } - case 131: + case 134: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.exp = nil } - case 132: + case 135: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.exp = yyDollar[2].exp } - case 133: + case 136: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.exp = nil } - case 134: + case 137: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.exp = yyDollar[2].exp } - case 135: + case 138: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.exp = nil } - case 136: + case 139: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.exp = yyDollar[2].exp } - case 137: + case 140: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.ordcols = nil } - case 138: + case 141: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.ordcols = yyDollar[3].ordcols } - case 139: + case 142: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.ids = nil } - case 140: + case 143: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.ids = yyDollar[4].ids } - case 141: + case 144: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.ordcols = []*OrdCol{{sel: yyDollar[1].sel, descOrder: yyDollar[2].opt_ord}} } - case 142: + case 145: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.ordcols = append(yyDollar[1].ordcols, &OrdCol{sel: yyDollar[3].sel, descOrder: yyDollar[4].opt_ord}) } - case 143: + case 146: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.opt_ord = false } - case 144: + case 147: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.opt_ord = false } - case 145: + case 148: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.opt_ord = true } - case 146: + case 149: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.id = "" } - case 147: + case 150: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.id = yyDollar[1].id } - case 148: + case 151: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.id = yyDollar[2].id } - case 149: + case 152: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.exp = yyDollar[1].exp } - case 150: + case 153: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.exp = yyDollar[1].binExp } - case 151: + case 154: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.exp = &NotBoolExp{exp: yyDollar[2].exp} } - case 152: + case 155: yyDollar = yyS[yypt-2 : yypt+1] { yyVAL.exp = &NumExp{left: &Integer{val: 0}, op: SUBSOP, right: yyDollar[2].exp} } - case 153: + case 156: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.exp = &LikeBoolExp{val: yyDollar[1].exp, notLike: yyDollar[2].boolean, pattern: yyDollar[4].exp} } - case 154: + case 157: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.exp = &ExistsBoolExp{q: (yyDollar[3].stmt).(DataSource)} } - case 155: + case 158: yyDollar = yyS[yypt-6 : yypt+1] { yyVAL.exp = &InSubQueryExp{val: yyDollar[1].exp, notIn: yyDollar[2].boolean, q: yyDollar[5].stmt.(*SelectStmt)} } - case 156: + case 159: yyDollar = yyS[yypt-6 : yypt+1] { yyVAL.exp = &InListExp{val: yyDollar[1].exp, notIn: yyDollar[2].boolean, values: yyDollar[5].values} } - case 157: + case 160: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.exp = yyDollar[1].sel } - case 158: + case 161: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.exp = yyDollar[1].value } - case 159: + case 162: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.exp = yyDollar[2].exp } - case 160: + case 163: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.exp = &Cast{val: yyDollar[1].exp, t: yyDollar[3].sqlType} } - case 161: + case 164: yyDollar = yyS[yypt-0 : yypt+1] { yyVAL.boolean = false } - case 162: + case 165: yyDollar = yyS[yypt-1 : yypt+1] { yyVAL.boolean = true } - case 163: + case 166: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &NumExp{left: yyDollar[1].exp, op: ADDOP, right: yyDollar[3].exp} } - case 164: + case 167: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &NumExp{left: yyDollar[1].exp, op: SUBSOP, right: yyDollar[3].exp} } - case 165: + case 168: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &NumExp{left: yyDollar[1].exp, op: DIVOP, right: yyDollar[3].exp} } - case 166: + case 169: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &NumExp{left: yyDollar[1].exp, op: MULTOP, right: yyDollar[3].exp} } - case 167: + case 170: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &BinBoolExp{left: yyDollar[1].exp, op: yyDollar[2].logicOp, right: yyDollar[3].exp} } - case 168: + case 171: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &CmpBoolExp{left: yyDollar[1].exp, op: yyDollar[2].cmpOp, right: yyDollar[3].exp} } - case 169: + case 172: yyDollar = yyS[yypt-3 : yypt+1] { yyVAL.binExp = &CmpBoolExp{left: yyDollar[1].exp, op: EQ, right: &NullValue{t: AnyType}} } - case 170: + case 173: yyDollar = yyS[yypt-4 : yypt+1] { yyVAL.binExp = &CmpBoolExp{left: yyDollar[1].exp, op: NE, right: &NullValue{t: AnyType}} diff --git a/embedded/sql/stmt.go b/embedded/sql/stmt.go index 15dd136e2f..de7859cae2 100644 --- a/embedded/sql/stmt.go +++ b/embedded/sql/stmt.go @@ -39,20 +39,34 @@ const ( catalogColumnPrefix = "CTL.COLUMN." // (key=CTL.COLUMN.{1}{tableID}{colID}{colTYPE}, value={(auto_incremental | nullable){maxLen}{colNAME}}) catalogIndexPrefix = "CTL.INDEX." // (key=CTL.INDEX.{1}{tableID}{indexID}, value={unique {colID1}(ASC|DESC)...{colIDN}(ASC|DESC)}) - RowPrefix = "R." // (key=R.{1}{tableID}{0}({null}({pkVal}{padding}{pkValLen})?)+, value={count (colID valLen val)+}) - + RowPrefix = "R." // (key=R.{1}{tableID}{0}({null}({pkVal}{padding}{pkValLen})?)+, value={count (colID valLen val)+}) MappedPrefix = "M." // (key=M.{tableID}{indexID}({null}({val}{padding}{valLen})?)*({pkVal}{padding}{pkValLen})+, value={count (colID valLen val)+}) ) -const DatabaseID = uint32(1) // deprecated but left to maintain backwards compatibility -const PKIndexID = uint32(0) +const ( + DatabaseID = uint32(1) // deprecated but left to maintain backwards compatibility + PKIndexID = uint32(0) +) const ( nullableFlag byte = 1 << iota autoIncrementFlag byte = 1 << iota ) -const revCol = "_rev" +const ( + revCol = "_rev" + txMetadataCol = "_tx_metadata" +) + +var reservedColumns = map[string]struct{}{ + revCol: {}, + txMetadataCol: {}, +} + +func isReservedCol(col string) bool { + _, ok := reservedColumns[col] + return ok +} type SQLValueType = string @@ -65,6 +79,7 @@ const ( Float64Type SQLValueType = "FLOAT" TimestampType SQLValueType = "TIMESTAMP" AnyType SQLValueType = "ANY" + JSONType SQLValueType = "JSON" ) func IsNumericType(t SQLValueType) bool { @@ -125,14 +140,15 @@ const ( ) const ( - NowFnCall string = "NOW" - UUIDFnCall string = "RANDOM_UUID" - DatabasesFnCall string = "DATABASES" - TablesFnCall string = "TABLES" - TableFnCall string = "TABLE" - UsersFnCall string = "USERS" - ColumnsFnCall string = "COLUMNS" - IndexesFnCall string = "INDEXES" + NowFnCall string = "NOW" + UUIDFnCall string = "RANDOM_UUID" + DatabasesFnCall string = "DATABASES" + TablesFnCall string = "TABLES" + TableFnCall string = "TABLE" + UsersFnCall string = "USERS" + ColumnsFnCall string = "COLUMNS" + IndexesFnCall string = "INDEXES" + JSONTypeOfFnCall string = "JSON_TYPEOF" ) type SQLStmt interface { @@ -461,6 +477,10 @@ func (stmt *CreateIndexStmt) execAt(ctx context.Context, tx *SQLTx, params map[s return nil, err } + if col.Type() == JSONType { + return nil, ErrCannotIndexJson + } + if variableSizedType(col.colType) && !tx.engine.lazyIndexConstraintValidation && (col.MaxLen() == 0 || col.MaxLen() > MaxKeyLen) { return nil, fmt.Errorf("%w: can not create index using column '%s'. Max key length for variable columns is %d", ErrLimitedKeyType, col.colName, MaxKeyLen) } @@ -1604,7 +1624,6 @@ func (n *NullValue) Compare(val TypedValue) (int, error) { if val.RawValue() == nil { return 0, nil } - return -1, nil } @@ -1671,7 +1690,7 @@ func (v *Integer) inferType(cols map[string]ColDescriptor, params map[string]SQL } func (v *Integer) requiresType(t SQLValueType, cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) error { - if t != IntegerType { + if t != IntegerType && t != JSONType { return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, IntegerType, t) } @@ -1707,6 +1726,11 @@ func (v *Integer) Compare(val TypedValue) (int, error) { return 1, nil } + if val.Type() == JSONType { + res, err := val.Compare(v) + return -res, err + } + if val.Type() == Float64Type { r, err := val.Compare(v) return r * -1, err @@ -1828,10 +1852,9 @@ func (v *Varchar) inferType(cols map[string]ColDescriptor, params map[string]SQL } func (v *Varchar) requiresType(t SQLValueType, cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) error { - if t != VarcharType { + if t != VarcharType && t != JSONType { return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, VarcharType, t) } - return nil } @@ -1864,6 +1887,11 @@ func (v *Varchar) Compare(val TypedValue) (int, error) { return 1, nil } + if val.Type() == JSONType { + res, err := val.Compare(v) + return -res, err + } + if val.Type() != VarcharType { return 0, ErrNotComparableValues } @@ -1968,10 +1996,9 @@ func (v *Bool) inferType(cols map[string]ColDescriptor, params map[string]SQLVal } func (v *Bool) requiresType(t SQLValueType, cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) error { - if t != BooleanType { + if t != BooleanType && t != JSONType { return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, BooleanType, t) } - return nil } @@ -2004,6 +2031,11 @@ func (v *Bool) Compare(val TypedValue) (int, error) { return 1, nil } + if val.Type() == JSONType { + res, err := val.Compare(v) + return -res, err + } + if val.Type() != BooleanType { return 0, ErrNotComparableValues } @@ -2116,10 +2148,9 @@ func (v *Float64) inferType(cols map[string]ColDescriptor, params map[string]SQL } func (v *Float64) requiresType(t SQLValueType, cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) error { - if t != Float64Type { + if t != Float64Type && t != JSONType { return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, Float64Type, t) } - return nil } @@ -2148,6 +2179,11 @@ func (v *Float64) RawValue() interface{} { } func (v *Float64) Compare(val TypedValue) (int, error) { + if val.Type() == JSONType { + res, err := val.Compare(v) + return -res, err + } + convVal, err := mayApplyImplicitConversion(val.RawValue(), Float64Type) if err != nil { return 0, err @@ -2187,6 +2223,10 @@ func (v *FnCall) inferType(cols map[string]ColDescriptor, params map[string]SQLV return UUIDType, nil } + if strings.ToUpper(v.fn) == JSONTypeOfFnCall { + return VarcharType, nil + } + return AnyType, fmt.Errorf("%w: unknown function %s", ErrIllegalArguments, v.fn) } @@ -2207,6 +2247,13 @@ func (v *FnCall) requiresType(t SQLValueType, cols map[string]ColDescriptor, par return nil } + if strings.ToUpper(v.fn) == JSONTypeOfFnCall { + if t != VarcharType { + return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, VarcharType, t) + } + return nil + } + return fmt.Errorf("%w: unkown function %s", ErrIllegalArguments, v.fn) } @@ -2241,6 +2288,26 @@ func (v *FnCall) reduce(tx *SQLTx, row *Row, implicitTable string) (TypedValue, return &UUID{val: uuid.New()}, nil } + if strings.ToUpper(v.fn) == JSONTypeOfFnCall { + if len(v.params) != 1 { + return nil, fmt.Errorf("%w: '%s' function expects %d arguments but %d were provided", ErrIllegalArguments, JSONTypeOfFnCall, 1, len(v.params)) + } + + v, err := v.params[0].reduce(tx, row, implicitTable) + if err != nil { + return nil, err + } + + if v.IsNull() { + return NewNull(AnyType), nil + } + + jsonVal, ok := v.(*JSON) + if !ok { + return nil, fmt.Errorf("%w: '%s' function expects an argument of type JSON", ErrIllegalArguments, JSONTypeOfFnCall) + } + return NewVarchar(jsonVal.primitiveType()), nil + } return nil, fmt.Errorf("%w: unkown function %s", ErrIllegalArguments, v.fn) } @@ -2392,7 +2459,6 @@ func (p *Param) substitute(params map[string]interface{}) (ValueExp, error) { return &Float64{val: v}, nil } } - return nil, ErrUnsupportedParameter } @@ -2723,6 +2789,22 @@ func (stmt *SelectStmt) Alias() string { return stmt.as } +func (stmt *SelectStmt) hasTxMetadata() bool { + for _, sel := range stmt.selectors { + switch s := sel.(type) { + case *ColSelector: + if s.col == txMetadataCol { + return true + } + case *JSONSelector: + if s.ColSelector.col == txMetadataCol { + return true + } + } + } + return false +} + func (stmt *SelectStmt) genScanSpecs(tx *SQLTx, params map[string]interface{}) (*ScanSpecs, error) { groupByCols, orderByCols := stmt.groupByOrdColumns(), stmt.orderBy @@ -2785,6 +2867,7 @@ func (stmt *SelectStmt) genScanSpecs(tx *SQLTx, params map[string]interface{}) ( Index: sortingIndex, rangesByColID: rangesByColID, IncludeHistory: tableRef.history, + IncludeTxMetadata: stmt.hasTxMetadata(), DescOrder: descOrder, groupBySortColumns: groupByCols, orderBySortCols: orderByCols, @@ -3102,7 +3185,6 @@ func (sel *ColSelector) resolve(implicitTable string) (aggFn, table, col string) if sel.table != "" { table = sel.table } - return "", table, sel.col } @@ -3409,9 +3491,21 @@ func (bexp *NumExp) reduce(tx *SQLTx, row *Row, implicitTable string) (TypedValu return nil, err } + vl = unwrapJSON(vl) + vr = unwrapJSON(vr) + return applyNumOperator(bexp.op, vl, vr) } +func unwrapJSON(v TypedValue) TypedValue { + if jsonVal, ok := v.(*JSON); ok { + if sv, isSimple := jsonVal.castToTypedValue(); isSimple { + return sv + } + } + return v +} + func (bexp *NumExp) reduceSelectors(row *Row, implicitTable string) ValueExp { return &NumExp{ op: bexp.op, @@ -3564,14 +3658,15 @@ func (bexp *LikeBoolExp) reduce(tx *SQLTx, row *Row, implicitTable string) (Type return nil, fmt.Errorf("error in 'LIKE' clause: %w", err) } - if rval.Type() != VarcharType { - return nil, fmt.Errorf("error in 'LIKE' clause: %w (expecting %s)", ErrInvalidTypes, VarcharType) - } - if rval.IsNull() { return &Bool{val: false}, nil } + rvalStr, ok := rval.RawValue().(string) + if !ok { + return nil, fmt.Errorf("error in 'LIKE' clause: %w (expecting %s)", ErrInvalidTypes, VarcharType) + } + rpattern, err := bexp.pattern.reduce(tx, row, implicitTable) if err != nil { return nil, fmt.Errorf("error in 'LIKE' clause: %w", err) @@ -3581,7 +3676,7 @@ func (bexp *LikeBoolExp) reduce(tx *SQLTx, row *Row, implicitTable string) (Type return nil, fmt.Errorf("error evaluating 'LIKE' clause: %w", ErrInvalidTypes) } - matched, err := regexp.MatchString(rpattern.RawValue().(string), rval.RawValue().(string)) + matched, err := regexp.MatchString(rpattern.RawValue().(string), rvalStr) if err != nil { return nil, fmt.Errorf("error in 'LIKE' clause: %w", err) } @@ -3834,16 +3929,16 @@ func updateRangeFor(colID uint32, val TypedValue, cmp CmpOperator, rangesByColID } func cmpSatisfiesOp(cmp int, op CmpOperator) bool { - switch cmp { - case 0: + switch { + case cmp == 0: { return op == EQ || op == LE || op == GE } - case -1: + case cmp < 0: { return op == NE || op == LT || op == LE } - case 1: + case cmp > 0: { return op == NE || op == GT || op == GE } @@ -3922,16 +4017,20 @@ func (bexp *BinBoolExp) reduce(tx *SQLTx, row *Row, implicitTable string) (Typed return nil, err } - vr, err := bexp.right.reduce(tx, row, implicitTable) - if err != nil { - return nil, err - } - bl, isBool := vl.(*Bool) if !isBool { return nil, fmt.Errorf("%w (expecting boolean value)", ErrInvalidValue) } + if (bexp.op == OR && bl.val) || (bexp.op == AND && !bl.val) { + return &Bool{val: bl.val}, nil + } + + vr, err := bexp.right.reduce(tx, row, implicitTable) + if err != nil { + return nil, err + } + br, isBool := vr.(*Bool) if !isBool { return nil, fmt.Errorf("%w (expecting boolean value)", ErrInvalidValue) diff --git a/embedded/sql/stmt_test.go b/embedded/sql/stmt_test.go index 2194505cc8..24d2b7d2d7 100644 --- a/embedded/sql/stmt_test.go +++ b/embedded/sql/stmt_test.go @@ -262,16 +262,18 @@ func TestRequiresTypeSimpleValueExp(t *testing.T) { cols["(mytable.payload)"] = ColDescriptor{Type: BLOBType} cols["COUNT(mytable.*)"] = ColDescriptor{Type: IntegerType} cols["(mytable.ft)"] = ColDescriptor{Type: Float64Type} + cols["(mytable.data)"] = ColDescriptor{Type: JSONType} params := make(map[string]SQLValueType) testCases := []struct { - exp ValueExp - cols map[string]ColDescriptor - params map[string]SQLValueType - implicitTable string - requiredType SQLValueType - expectedError error + exp ValueExp + cols map[string]ColDescriptor + params map[string]SQLValueType + implicitTable string + requiredType SQLValueType + expectedInferredType SQLValueType + expectedError error }{ { exp: &NullValue{t: AnyType}, @@ -313,6 +315,15 @@ func TestRequiresTypeSimpleValueExp(t *testing.T) { requiredType: VarcharType, expectedError: ErrInvalidTypes, }, + { + exp: &Integer{}, + cols: cols, + params: params, + implicitTable: "mytable", + requiredType: JSONType, + expectedInferredType: IntegerType, + expectedError: nil, + }, { exp: &Varchar{}, cols: cols, @@ -321,6 +332,15 @@ func TestRequiresTypeSimpleValueExp(t *testing.T) { requiredType: VarcharType, expectedError: nil, }, + { + exp: &Varchar{}, + cols: cols, + params: params, + implicitTable: "mytable", + requiredType: JSONType, + expectedInferredType: VarcharType, + expectedError: nil, + }, { exp: &Varchar{}, cols: cols, @@ -337,6 +357,15 @@ func TestRequiresTypeSimpleValueExp(t *testing.T) { requiredType: BooleanType, expectedError: nil, }, + { + exp: &Bool{}, + cols: cols, + params: params, + implicitTable: "mytable", + requiredType: JSONType, + expectedInferredType: BooleanType, + expectedError: nil, + }, { exp: &Bool{}, cols: cols, @@ -361,6 +390,68 @@ func TestRequiresTypeSimpleValueExp(t *testing.T) { requiredType: IntegerType, expectedError: ErrInvalidTypes, }, + { + exp: &JSON{}, + cols: cols, + params: params, + requiredType: JSONType, + implicitTable: "mytable", + expectedError: nil, + }, + { + exp: &JSON{val: "some-string"}, + cols: cols, + params: params, + requiredType: VarcharType, + expectedInferredType: JSONType, + implicitTable: "mytable", + expectedError: nil, + }, + { + exp: &JSON{val: int64(10)}, + cols: cols, + params: params, + requiredType: Float64Type, + expectedInferredType: JSONType, + implicitTable: "mytable", + expectedError: nil, + }, + { + exp: &JSON{val: float64(10.5)}, + cols: cols, + params: params, + requiredType: IntegerType, + expectedInferredType: JSONType, + implicitTable: "mytable", + expectedError: ErrInvalidTypes, + }, + { + exp: &JSON{val: true}, + cols: cols, + params: params, + requiredType: BooleanType, + expectedInferredType: JSONType, + implicitTable: "mytable", + expectedError: nil, + }, + { + exp: &JSON{val: nil}, + cols: cols, + params: params, + requiredType: AnyType, + expectedInferredType: JSONType, + implicitTable: "mytable", + expectedError: nil, + }, + { + exp: &JSON{val: int64(10)}, + cols: cols, + params: params, + requiredType: IntegerType, + expectedInferredType: JSONType, + implicitTable: "mytable", + expectedError: nil, + }, { exp: &NotBoolExp{exp: &Bool{val: true}}, cols: cols, @@ -424,9 +515,14 @@ func TestRequiresTypeSimpleValueExp(t *testing.T) { require.ErrorIs(t, err, tc.expectedError, fmt.Sprintf("failed on iteration %d", i)) if tc.expectedError == nil { + expectedInferredType := tc.expectedInferredType + if expectedInferredType == "" { + expectedInferredType = tc.requiredType + } + it, err := tc.exp.inferType(tc.cols, params, tc.implicitTable) require.NoError(t, err) - require.Equal(t, tc.requiredType, it) + require.Equal(t, expectedInferredType, it) } } } @@ -667,7 +763,7 @@ func TestLikeBoolExpEdgeCases(t *testing.T) { err = exp.requiresType(BooleanType, nil, nil, "") require.ErrorIs(t, err, ErrInvalidTypes) - v := &NullValue{} + v := &Integer{} row := &Row{ ValuesByPosition: []TypedValue{v}, @@ -768,7 +864,7 @@ func TestIsConstant(t *testing.T) { require.False(t, (&ExistsBoolExp{}).isConstant()) } -func TestTimestmapType(t *testing.T) { +func TestTimestamapType(t *testing.T) { ts := &Timestamp{val: time.Date(2021, 12, 6, 11, 53, 0, 0, time.UTC)} @@ -824,6 +920,167 @@ func TestTimestmapType(t *testing.T) { require.NoError(t, err) } +func TestJSONType(t *testing.T) { + js := &JSON{val: float64(10)} + + require.True(t, js.isConstant()) + require.False(t, js.IsNull()) + + it, err := js.inferType(map[string]ColDescriptor{}, map[string]string{}, "") + require.NoError(t, err) + require.Equal(t, JSONType, it) + + v, err := js.substitute(map[string]interface{}{}) + require.NoError(t, err) + require.Equal(t, js, v) + + v, err = js.reduce(nil, nil, "") + require.NoError(t, err) + require.Equal(t, js, v) + + v = js.reduceSelectors(&Row{}, "") + require.Equal(t, js, v) + + err = js.selectorRanges(&Table{}, "", map[string]interface{}{}, map[uint32]*typedValueRange{}) + require.NoError(t, err) + + t.Run("test comparison functions", func(t *testing.T) { + type test struct { + a TypedValue + b TypedValue + res int + expectedError error + } + + tests := []test{ + { + a: NewJson(10.5), + b: NewJson(10.5), + }, + { + a: NewJson(map[string]interface{}{}), + b: NewJson(map[string]interface{}{}), + expectedError: ErrNotComparableValues, + }, + { + a: NewJson(10.5), + b: NewFloat64(9.5), + res: 1, + }, + { + a: NewJson(true), + b: NewBool(true), + res: 0, + }, + { + a: NewJson("test"), + b: NewVarchar("test"), + res: 0, + }, + { + a: NewJson(int64(2)), + b: NewInteger(8), + res: -1, + }, + { + a: NewJson(nil), + b: NewNull(JSONType), + res: 0, + }, + { + a: NewJson(nil), + b: NewNull(AnyType), + res: 0, + }, + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("compare %s to %s", tc.a.Type(), tc.b.Type()), func(t *testing.T) { + res, err := tc.a.Compare(tc.b) + if tc.expectedError != nil { + require.ErrorIs(t, err, ErrNotComparableValues) + } else { + require.NoError(t, err) + require.Equal(t, tc.res, res) + } + + res1, err := tc.b.Compare(tc.a) + if tc.expectedError != nil { + require.ErrorIs(t, err, ErrNotComparableValues) + } else { + require.NoError(t, err) + require.Equal(t, res, -res1) + } + }) + } + }) + + t.Run("test casts", func(t *testing.T) { + type test struct { + src TypedValue + dst TypedValue + } + + cases := []test{ + { + src: &NullValue{t: JSONType}, + dst: &JSON{val: nil}, + }, + { + src: &NullValue{t: AnyType}, + dst: &JSON{val: nil}, + }, + { + src: &JSON{val: nil}, + dst: &NullValue{t: AnyType}, + }, + { + src: &JSON{val: 10.5}, + dst: &Float64{val: 10.5}, + }, + { + src: &Float64{val: 10.5}, + dst: &JSON{val: 10.5}, + }, + { + src: &JSON{val: 10.5}, + dst: &Integer{val: 10}, + }, + { + src: &Integer{val: 10}, + dst: &JSON{val: int64(10)}, + }, + { + src: &JSON{val: true}, + dst: &Bool{val: true}, + }, + { + src: &Bool{val: true}, + dst: &JSON{val: true}, + }, + { + src: &JSON{val: "test"}, + dst: &Varchar{val: `"test"`}, + }, + { + src: &Varchar{val: `{"name": "John Doe"}`}, + dst: &JSON{val: map[string]interface{}{"name": "John Doe"}}, + }, + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("cast %s to %s", tc.src.Type(), tc.dst.Type()), func(t *testing.T) { + conv, err := getConverter(tc.src.Type(), tc.dst.Type()) + require.NoError(t, err) + + converted, err := conv(tc.src) + require.NoError(t, err) + require.Equal(t, converted, tc.dst) + }) + } + }) +} + func TestUnionSelectErrors(t *testing.T) { t.Run("fail on creating union reader", func(t *testing.T) { reader1 := &dummyRowReader{ @@ -1095,4 +1352,7 @@ func TestTypedValueString(t *testing.T) { avg := &AVGValue{s: &Float64{val: 10}, c: 4} require.Equal(t, "2.5", avg.String()) + + jsVal := &JSON{val: map[string]interface{}{"name": "John Doe"}} + require.Equal(t, jsVal.String(), `{"name":"John Doe"}`) } diff --git a/embedded/sql/type_conversion.go b/embedded/sql/type_conversion.go index 6beaa0f988..623911b79f 100644 --- a/embedded/sql/type_conversion.go +++ b/embedded/sql/type_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package sql import ( + "encoding/json" "fmt" "strconv" "time" @@ -28,12 +29,20 @@ type converterFunc func(TypedValue) (TypedValue, error) func getConverter(src, dst SQLValueType) (converterFunc, error) { if src == dst { + if src == JSONType { + return jsonConverted(dst), nil + } + return func(tv TypedValue) (TypedValue, error) { return tv, nil }, nil } if src == AnyType { + if dst == JSONType { + return jsonConverted(dst), nil + } + return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { return &NullValue{t: dst}, nil @@ -43,7 +52,6 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { } if dst == TimestampType { - if src == IntegerType { return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { @@ -93,11 +101,9 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { "%w: only INTEGER and VARCHAR types can be cast as TIMESTAMP", ErrUnsupportedCast, ) - } if dst == Float64Type { - if src == IntegerType { return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { @@ -125,15 +131,30 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { }, nil } + if src == JSONType { + return jsonConverted(dst), nil + } + return nil, fmt.Errorf( "%w: only INTEGER and VARCHAR types can be cast as FLOAT", ErrUnsupportedCast, ) + } + + if dst == BooleanType { + if src == JSONType { + return jsonConverted(dst), nil + } + return nil, fmt.Errorf( + "%w: cannot cast %s to %s", + ErrUnsupportedCast, + src, + dst, + ) } if dst == IntegerType { - if src == Float64Type { return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { @@ -161,15 +182,17 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { }, nil } + if src == JSONType { + return jsonConverted(dst), nil + } + return nil, fmt.Errorf( "%w: only INTEGER and VARCHAR types can be cast as INTEGER", ErrUnsupportedCast, ) - } if dst == UUIDType { - if src == VarcharType { return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { @@ -216,11 +239,9 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { "%w: only BLOB and VARCHAR types can be cast as UUID", ErrUnsupportedCast, ) - } if dst == BLOBType { - if src == VarcharType { return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { @@ -253,7 +274,6 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { } if dst == VarcharType { - if src == UUIDType { return func(val TypedValue) (TypedValue, error) { if val.RawValue() == nil { @@ -266,16 +286,81 @@ func getConverter(src, dst SQLValueType) (converterFunc, error) { }, nil } + if src == JSONType { + return jsonConverted(dst), nil + } + return nil, fmt.Errorf( "%w: only UUID type can be cast as VARCHAR", ErrUnsupportedCast, ) + } + + if dst == JSONType { + return func(tv TypedValue) (TypedValue, error) { + if tv.RawValue() == nil { + return &NullValue{t: JSONType}, nil + } + + switch tv.Type() { + case Float64Type, IntegerType, BooleanType, AnyType: + return &JSON{val: tv.RawValue()}, nil + case VarcharType: + var x interface{} + err := json.Unmarshal([]byte(tv.String()), &x) + return &JSON{val: x}, err + } + + return nil, fmt.Errorf( + "%w: can not cast %s value as %s", + ErrUnsupportedCast, + tv.Type(), + JSONType, + ) + }, nil + } + if dst == AnyType && src == JSONType { + return func(tv TypedValue) (TypedValue, error) { + if !tv.IsNull() { + return &NullValue{t: AnyType}, nil + } + return nil, ErrInvalidValue + }, nil } return nil, fmt.Errorf( "%w: can not cast %s value as %s", ErrUnsupportedCast, - src, dst, + src, + dst, ) } + +func jsonConverted(t SQLValueType) converterFunc { + return func(val TypedValue) (TypedValue, error) { + if val.IsNull() { + return &JSON{val: nil}, nil + } + + jsonVal := val.(*JSON) + if t == VarcharType { + return NewVarchar(jsonVal.String()), nil + } + + val, ok := jsonVal.castToTypedValue() + if !ok { + return nil, fmt.Errorf( + "%w: can not cast JSON as %s", + ErrUnsupportedCast, + t, + ) + } + + conv, err := getConverter(val.Type(), t) + if err != nil { + return nil, err + } + return conv(val) + } +} diff --git a/pkg/api/schema/docs.md b/pkg/api/schema/docs.md index 6273e640e3..7b7dc1391c 100644 --- a/pkg/api/schema/docs.md +++ b/pkg/api/schema/docs.md @@ -916,7 +916,7 @@ DualProofV2 contains inclusion and consistency proofs | ----- | ---- | ----- | ----------- | | flushThreshold | [NullableUint32](#immudb.schema.NullableUint32) | | Number of new index entries between disk flushes | | syncThreshold | [NullableUint32](#immudb.schema.NullableUint32) | | Number of new index entries between disk flushes with file sync | -| cacheSize | [NullableUint32](#immudb.schema.NullableUint32) | | Size of the Btree node cache | +| cacheSize | [NullableUint32](#immudb.schema.NullableUint32) | | Size of the Btree node cache in bytes | | maxNodeSize | [NullableUint32](#immudb.schema.NullableUint32) | | Max size of a single Btree node in bytes | | maxActiveSnapshots | [NullableUint32](#immudb.schema.NullableUint32) | | Maximum number of active btree snapshots | | renewSnapRootAfter | [NullableUint64](#immudb.schema.NullableUint64) | | Time in milliseconds between the most recent DB snapshot is automatically renewed | diff --git a/pkg/api/schema/row_value.go b/pkg/api/schema/row_value.go index df1f20c89b..326f6b0561 100644 --- a/pkg/api/schema/row_value.go +++ b/pkg/api/schema/row_value.go @@ -184,7 +184,6 @@ func RenderValueAsByte(op isSQLValue_Value) []byte { return []byte(strconv.FormatFloat(float64(v.F), 'f', -1, 64)) } } - return []byte(fmt.Sprintf("%v", op)) } @@ -223,6 +222,5 @@ func RawValue(v *SQLValue) interface{} { return tv.F } } - return nil } diff --git a/pkg/api/schema/schema.pb.go b/pkg/api/schema/schema.pb.go index 0627714eed..495b3d84d2 100644 --- a/pkg/api/schema/schema.pb.go +++ b/pkg/api/schema/schema.pb.go @@ -6157,7 +6157,7 @@ type IndexNullableSettings struct { FlushThreshold *NullableUint32 `protobuf:"bytes,1,opt,name=flushThreshold,proto3" json:"flushThreshold,omitempty"` // Number of new index entries between disk flushes with file sync SyncThreshold *NullableUint32 `protobuf:"bytes,2,opt,name=syncThreshold,proto3" json:"syncThreshold,omitempty"` - // Size of the Btree node cache + // Size of the Btree node cache in bytes CacheSize *NullableUint32 `protobuf:"bytes,3,opt,name=cacheSize,proto3" json:"cacheSize,omitempty"` // Max size of a single Btree node in bytes MaxNodeSize *NullableUint32 `protobuf:"bytes,4,opt,name=maxNodeSize,proto3" json:"maxNodeSize,omitempty"` diff --git a/pkg/api/schema/schema.swagger.json b/pkg/api/schema/schema.swagger.json index d44a4ef413..e726c20157 100644 --- a/pkg/api/schema/schema.swagger.json +++ b/pkg/api/schema/schema.swagger.json @@ -2635,7 +2635,7 @@ }, "cacheSize": { "$ref": "#/definitions/schemaNullableUint32", - "title": "Size of the Btree node cache" + "title": "Size of the Btree node cache in bytes" }, "maxNodeSize": { "$ref": "#/definitions/schemaNullableUint32", diff --git a/pkg/api/schema/sql.go b/pkg/api/schema/sql.go index be1edd0439..74999b3189 100644 --- a/pkg/api/schema/sql.go +++ b/pkg/api/schema/sql.go @@ -152,6 +152,8 @@ func TypedValueToRowValue(tv sql.TypedValue) *SQLValue { { return &SQLValue{Value: &SQLValue_F{F: tv.RawValue().(float64)}} } + case sql.JSONType: + return &SQLValue{Value: &SQLValue_S{S: tv.String()}} } return nil } diff --git a/pkg/client/sql.go b/pkg/client/sql.go index acd1e99a19..1448525800 100644 --- a/pkg/client/sql.go +++ b/pkg/client/sql.go @@ -463,6 +463,10 @@ func (it *rowReader) Read() (Row, error) { return nil, it.err } + if it.nextRow < 0 { + return nil, errors.New("Read called without calling Next") + } + protoRow := it.rows[it.nextRow] for i, protoVal := range protoRow.Values { val := schema.RawValue(protoVal) diff --git a/pkg/database/database.go b/pkg/database/database.go index 3c6fcdc0ee..21fea86b33 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -226,7 +226,8 @@ func OpenDB(dbName string, multidbHandler sql.MultiDBHandler, opts *Options, log sqlOpts := sql.DefaultOptions(). WithPrefix([]byte{SQLPrefix}). - WithMultiDBHandler(multidbHandler) + WithMultiDBHandler(multidbHandler). + WithParseTxMetadataFunc(parseTxMetadata) dbi.sqlEngine, err = sql.NewEngine(dbi.st, sqlOpts) if err != nil { @@ -257,6 +258,19 @@ func OpenDB(dbName string, multidbHandler sql.MultiDBHandler, opts *Options, log return dbi, nil } +func parseTxMetadata(data []byte) (map[string]interface{}, error) { + md := schema.Metadata{} + if err := md.Unmarshal(data); err != nil { + return nil, err + } + + meta := make(map[string]interface{}, len(md)) + for k, v := range md { + meta[k] = v + } + return meta, nil +} + func (d *db) Path() string { return filepath.Join(d.options.GetDBRootPath(), d.GetName()) } @@ -336,7 +350,8 @@ func NewDB(dbName string, multidbHandler sql.MultiDBHandler, opts *Options, log sqlOpts := sql.DefaultOptions(). WithPrefix([]byte{SQLPrefix}). - WithMultiDBHandler(multidbHandler) + WithMultiDBHandler(multidbHandler). + WithParseTxMetadataFunc(parseTxMetadata) dbi.Logger.Infof("loading sql-engine for database '%s' {replica = %v}...", dbName, opts.replica) diff --git a/pkg/integration/sql/sql_test.go b/pkg/integration/sql/sql_test.go index 2fbe43e304..6a14118b9f 100644 --- a/pkg/integration/sql/sql_test.go +++ b/pkg/integration/sql/sql_test.go @@ -18,6 +18,7 @@ package integration import ( "context" + "encoding/json" "fmt" "testing" @@ -276,6 +277,7 @@ func TestImmuClient_SQLQueryReader(t *testing.T) { CREATE TABLE test_table ( id INTEGER AUTO_INCREMENT, value INTEGER, + data JSON, PRIMARY KEY (id) ); @@ -283,14 +285,25 @@ func TestImmuClient_SQLQueryReader(t *testing.T) { require.NoError(t, err) for n := 0; n < 10; n++ { - _, err := client.SQLExec(ctx, "INSERT INTO test_table(value) VALUES (@value)", map[string]interface{}{"value": n + 10}) + name := fmt.Sprintf("name%d", n) + + _, err := client.SQLExec( + ctx, + "INSERT INTO test_table(value, data) VALUES (@value, @data)", + map[string]interface{}{ + "value": n + 10, + "data": fmt.Sprintf(`{"name": "%s"}`, name), + }) require.NoError(t, err) } reader, err := client.SQLQueryReader(ctx, "SELECT * FROM test_table WHERE value < 0", nil) require.NoError(t, err) - require.False(t, reader.Next()) + _, err = reader.Read() + require.Error(t, err) + + require.False(t, reader.Next()) _, err = reader.Read() require.ErrorIs(t, err, sql.ErrNoMoreRows) @@ -305,15 +318,24 @@ func TestImmuClient_SQLQueryReader(t *testing.T) { require.Equal(t, cols[0].Type, sql.IntegerType) require.Equal(t, cols[1].Name, "(test_table.value)") require.Equal(t, cols[1].Type, sql.IntegerType) + require.Equal(t, cols[2].Name, "(test_table.data)") + require.Equal(t, cols[2].Type, sql.JSONType) n := 0 for reader.Next() { row, err := reader.Read() require.NoError(t, err) - require.Len(t, row, 2) + require.Len(t, row, 3) + + name := fmt.Sprintf("name%d", n) + + var data interface{} + err = json.Unmarshal([]byte(row[2].(string)), &data) + require.NoError(t, err) require.Equal(t, int64(n+1), row[0]) require.Equal(t, int64(n+10), row[1]) + require.Equal(t, map[string]interface{}{"name": name}, data) n++ } @@ -499,3 +521,59 @@ func TestImmuClient_SQL_Errors(t *testing.T) { }, "table1", []*schema.SQLValue{{Value: &schema.SQLValue_N{N: 1}}}) require.ErrorIs(t, err, ic.ErrNotConnected) } + +func TestQueryTxMetadata(t *testing.T) { + options := server.DefaultOptions(). + WithDir(t.TempDir()). + WithLogRequestMetadata(true) + + bs := servertest.NewBufconnServer(options) + + bs.Start() + defer bs.Stop() + + client, err := bs.NewAuthenticatedClient(ic.DefaultOptions().WithDir(t.TempDir())) + require.NoError(t, err) + defer client.CloseSession(context.Background()) + + _, err = client.SQLExec( + context.Background(), + `CREATE TABLE mytable( + id INTEGER, + data JSON, + + PRIMARY KEY (id) + )`, + nil, + ) + require.NoError(t, err) + + _, err = client.SQLExec( + context.Background(), + `INSERT INTO mytable(id, data) VALUES (1, '{"name": "John Doe"}')`, + nil, + ) + require.NoError(t, err) + + it, err := client.SQLQueryReader( + context.Background(), + "SELECT _tx_metadata FROM mytable", + nil, + ) + require.NoError(t, err) + + require.True(t, it.Next()) + + row, err := it.Read() + require.NoError(t, err) + + var md map[string]interface{} + err = json.Unmarshal([]byte(row[0].(string)), &md) + require.NoError(t, err) + + require.Equal( + t, + map[string]interface{}{"usr": "immudb", "ip": "bufconn"}, + md, + ) +} diff --git a/pkg/pgsql/server/bmessages/data_row.go b/pkg/pgsql/server/bmessages/data_row.go index d251a5aba6..02e4bb1f6f 100644 --- a/pkg/pgsql/server/bmessages/data_row.go +++ b/pkg/pgsql/server/bmessages/data_row.go @@ -65,6 +65,12 @@ func DataRow(rows []*sql.Row, colNumb int, ResultColumnFormatCodes []int16) []by value = make([]byte, 8) binary.BigEndian.PutUint64(value, uint64(rv.(int64))) } + case sql.JSONType: + { + jsonStr := val.String() + binary.BigEndian.PutUint32(valueLength, uint32(len(jsonStr))) + value = []byte(jsonStr) + } case sql.VarcharType: { s := rv.(string) diff --git a/pkg/pgsql/server/pgmeta/pg_type.go b/pkg/pgsql/server/pgmeta/pg_type.go index 599d0429fc..7a2839290a 100644 --- a/pkg/pgsql/server/pgmeta/pg_type.go +++ b/pkg/pgsql/server/pgmeta/pg_type.go @@ -19,6 +19,8 @@ package pgmeta import ( "errors" "fmt" + + "github.com/codenotary/immudb/embedded/sql" ) const ( @@ -37,13 +39,14 @@ var ErrInvalidPgsqlProtocolVersion = errors.New("invalid pgsql protocol version" // First int is the oid value (retrieved with select * from pg_type;) // Second int is the length of the value. -1 for dynamic. var PgTypeMap = map[string][]int{ - "BOOLEAN": {16, 1}, //bool - "BLOB": {17, -1}, //bytea - "TIMESTAMP": {20, 8}, //int8 - "INTEGER": {20, 8}, //int8 - "VARCHAR": {25, -1}, //text - "UUID": {2950, 16}, //uuid - "FLOAT": {701, 8}, //double-precision floating point number + sql.BooleanType: {16, 1}, //bool + sql.BLOBType: {17, -1}, //bytea + sql.TimestampType: {20, 8}, //int8 + sql.IntegerType: {20, 8}, //int8 + sql.VarcharType: {25, -1}, //text + sql.UUIDType: {2950, 16}, //uuid + sql.Float64Type: {701, 8}, //double-precision floating point number + sql.JSONType: {114, -1}, //json } const PgSeverityError = "ERROR" diff --git a/pkg/pgsql/server/stmts_handler.go b/pkg/pgsql/server/stmts_handler.go index dd05343158..0a5e40ea22 100644 --- a/pkg/pgsql/server/stmts_handler.go +++ b/pkg/pgsql/server/stmts_handler.go @@ -49,6 +49,7 @@ func (s *session) isEmulableInternally(statement string) interface{} { } return nil } + func (s *session) tryToHandleInternally(command interface{}) error { switch cmd := command.(type) { case *version: