Skip to content

Commit

Permalink
Merge pull request #8 from gcpug/use-tenntenn-jsonschema
Browse files Browse the repository at this point in the history 
Use tenntenn jsonschema
  • Loading branch information
@tenntenn
tenntenn authored Mar 5, 2019
2 parents e5a7d26 + 0d5f7f3 commit 3094ca3
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 53 deletions.
7 changes: 1 addition & 6 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,7 @@ module github.com/gcpug/hake
require (
cloud.google.com/go v0.35.1
github.com/golang/protobuf v1.2.0
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 // indirect
github.com/minio/minio v0.0.0-20190212015826-b8955fe57772
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/tenntenn/jsonschema v0.0.2
github.com/xeipuuv/gojsonschema v1.1.0
golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045 // indirect
google.golang.org/api v0.1.0
google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922
)
10 changes: 4 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
Expand All @@ -53,8 +51,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/minio/minio v0.0.0-20190212015826-b8955fe57772 h1:fLv8dS1IEDsjtpvfTUIxKn3nSvyqY9xykH5RSl3nCPI=
github.com/minio/minio v0.0.0-20190212015826-b8955fe57772/go.mod h1:lXcp05uxYaW99ebgI6ZKIGYU7tqZkM5xSsG0xRt4VIU=
github.com/minio/minio v0.0.0-20190216002119-b6c00405ec5c h1:csjhUhUbwprAuwFxy7sCm94CvV1s9qY8Y1gM5mpC4jA=
github.com/minio/minio v0.0.0-20190216002119-b6c00405ec5c/go.mod h1:lXcp05uxYaW99ebgI6ZKIGYU7tqZkM5xSsG0xRt4VIU=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
Expand Down Expand Up @@ -93,6 +91,8 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tenntenn/jsonschema v0.0.1 h1:eIKG8Xiw3nWVILPd3UuGww88j80vGEbSSYNGdcIy89c=
github.com/tenntenn/jsonschema v0.0.1/go.mod h1:1b70ZBDcPdpWbr6rkPPuwyY/Lc+t0zYP/p+3DBTS+M0=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
Expand All @@ -102,8 +102,6 @@ github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4m
go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045 h1:Pn8fQdvx+z1avAi7fdM2kRYWQNxGlavNDSyzrQg2SsU=
golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand Down
15 changes: 9 additions & 6 deletions json_column.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"cloud.google.com/go/civil"
"cloud.google.com/go/spanner"
structpb "github.com/golang/protobuf/ptypes/struct"
"github.com/tenntenn/jsonschema"
gspanner "google.golang.org/genproto/googleapis/spanner/v1"
)

Expand Down Expand Up @@ -93,7 +94,7 @@ func (c *JSONColumn) marshalList(t *gspanner.Type, l *structpb.ListValue) ([]int
return vs, nil
}

func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...JSONSchemaOption) error {
func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...jsonschema.Option) error {

switch t.Code {
default:
Expand Down Expand Up @@ -124,15 +125,17 @@ func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...JSONSchem
}

for i := range options {
if err := (options[i])(o); err != nil {
var err error
o, err = (options[i])(o)
if err != nil {
return err
}
}

return nil
}

func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, options ...JSONSchemaOption) error {
func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, options ...jsonschema.Option) error {

required := make([]string, len(t.Fields))
properties := make(map[string]interface{}, len(t.Fields))
Expand All @@ -146,9 +149,9 @@ func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, opt
ref: path.Join(parent.Ref(), "properties", f.Name),
}

opts := make([]JSONSchemaOption, len(options)+1)
opts := make([]jsonschema.Option, len(options)+1)
copy(opts, options)
opts[len(opts)-1] = ByJSONReference(o.Ref(), PropertyOrder(i))
opts[len(opts)-1] = jsonschema.ByReference(o.Ref(), jsonschema.PropertyOrder(i))

if err := c.schema(o, f.Type, opts...); err != nil {
return err
Expand All @@ -164,7 +167,7 @@ func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, opt
return nil
}

func (c *JSONColumn) schemaArray(parent JSONObject, t *gspanner.Type, options ...JSONSchemaOption) error {
func (c *JSONColumn) schemaArray(parent JSONObject, t *gspanner.Type, options ...jsonschema.Option) error {

o := &mapJSONObject{
m: map[string]interface{}{},
Expand Down
63 changes: 30 additions & 33 deletions json_row.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"path"

"cloud.google.com/go/spanner"
"github.com/minio/minio/pkg/wildcard"
"github.com/tenntenn/jsonschema"
)

// JSONRow is an encodable type of spanner.Row.
Expand Down Expand Up @@ -55,30 +55,8 @@ func (o *mapJSONObject) Ref() string {
return o.ref
}

// JSONSchemaOption is options for JSON Schema.
type JSONSchemaOption func(o JSONObject) error

// ByJSONReference explicits refrence of adding option.
// It only supports refs which begins "#/".
func ByJSONReference(pattern string, opt JSONSchemaOption) JSONSchemaOption {
return func(o JSONObject) error {
if wildcard.MatchSimple(pattern, o.Ref()) {
return opt(o)
}
return nil
}
}

// PropertyOrder is add propertyOrder to schema.
func PropertyOrder(order int) JSONSchemaOption {
return func(o JSONObject) error {
o.Set("propertyOrder", order)
return nil
}
}

// Schema writes JSON Schema of the row to writer w.
func (r *JSONRow) Schema(w io.Writer, options ...JSONSchemaOption) error {
func (r *JSONRow) JSONSchema(w io.Writer, options ...jsonschema.Option) error {
type colSchema struct {
Name string
Schema string
Expand All @@ -98,9 +76,9 @@ func (r *JSONRow) Schema(w io.Writer, options ...JSONSchemaOption) error {
ref: path.Join("#/properties", names[i]),
}

opts := make([]JSONSchemaOption, len(options)+1)
opts := make([]jsonschema.Option, len(options)+1)
copy(opts, options)
opts[len(opts)-1] = ByJSONReference(o.Ref(), PropertyOrder(i))
opts[len(opts)-1] = jsonschema.ByReference(o.Ref(), jsonschema.PropertyOrder(i))

if err := (*JSONColumn)(&col).schema(o, col.Type, opts...); err != nil {
return err
Expand Down Expand Up @@ -134,16 +112,35 @@ func (r *JSONRow) Schema(w io.Writer, options ...JSONSchemaOption) error {
return nil
}

// JSONRows convert []*spanner.Row to []*Row.
func JSONRows(rows []*spanner.Row) []*JSONRow {
if rows == nil {
// JSONRows is an encodable type of []*spanner.Row.
type JSONRows []*spanner.Row

var _ json.Marshaler = (JSONRows)(nil)

// At returns ith row as *JSONRow.
func (rs JSONRows) At(i int) *JSONRow {
return (*JSONRow)(rs[i])
}

func (rs JSONRows) toJSONRowSlice() []*JSONRow {
if rs == nil {
return nil
}

rs := make([]*JSONRow, len(rows))
for i := range rows {
rs[i] = (*JSONRow)(rows[i])
rows := make([]*JSONRow, len(rs))
for i := range rs {
rows[i] = rs.At(i)
}

return rs
return rows
}

// MarshalJSON implements json.Marshaler
func (rs JSONRows) MarshalJSON() ([]byte, error) {
return json.Marshal(rs.toJSONRowSlice())
}

// Schema writes JSON Schema of the rows to writer w.
func (rs JSONRows) JSONSchema(w io.Writer, options ...jsonschema.Option) error {
return jsonschema.Generate(w, rs.toJSONRowSlice(), options...)
}
70 changes: 68 additions & 2 deletions json_row_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,18 @@ func TestRows(t *testing.T) {
}
}

func TestJSONRow_Schema(t *testing.T) {
type noopFormatCheker struct{}

func (noopFormatCheker) IsFormat(_ string) bool {
return true
}

func init() {
gojsonschema.FormatCheckers.Add("datetime", noopFormatCheker{})
gojsonschema.FormatCheckers.Add("textarea", noopFormatCheker{})
}

func TestJSONRow_JSONSchema(t *testing.T) {

type T struct {
N int
Expand All @@ -83,7 +94,7 @@ func TestJSONRow_Schema(t *testing.T) {
tt := tt
t.Run(tt.name, func(t *testing.T) {
var got bytes.Buffer
err := (*JSONRow)(tt.row).Schema(&got)
err := (*JSONRow)(tt.row).JSONSchema(&got)
switch {
case tt.isErr && err == nil:
t.Errorf("expected error does not occur")
Expand All @@ -109,3 +120,58 @@ func TestJSONRow_Schema(t *testing.T) {
})
}
}

func TestJSONRows_JSONSchema(t *testing.T) {

type T struct {
N int
S string
}

type NT struct {
T T
}

cases := []struct {
name string
rows []*spanner.Row
isErr bool
}{
{"int", rows(t, []R{{"col1", 100}}), false},
{"int int", rows(t, []R{{"col1", 100}, {"col1", 200}}), false},
{"int string", rows(t, []R{{"col1", 100, "col2", "string"}}), false},
{"nested struct", rows(t, []R{{"col1", 100, "col2", T{N: 100, S: ""}}}), false},
{"timestamp", rows(t, []R{{"col1", 100, "col2", timestamp(t, "2002-10-02T10:00:00Z")}}), false},
{"bytes", rows(t, []R{{"col1", []byte("test")}}), false},
}

for _, tt := range cases {
tt := tt
t.Run(tt.name, func(t *testing.T) {
var got bytes.Buffer
err := JSONRows(tt.rows).JSONSchema(&got)
switch {
case tt.isErr && err == nil:
t.Errorf("expected error does not occur")
case !tt.isErr && err != nil:
t.Errorf("unexpected error %v", err)
}

l := gojsonschema.NewStringLoader(got.String())
s, err := gojsonschema.NewSchema(l)
if err != nil {
t.Fatalf("unexpected error %v", err)
}

rowJSON := toJSON(t, JSONRows(tt.rows))
r, err := s.Validate(gojsonschema.NewStringLoader(rowJSON))
if err != nil {
t.Fatalf("unexpected error %v", err)
}

if !r.Valid() {
t.Errorf("invalid JSON Schema: %s", got.String())
}
})
}
}

0 comments on commit 3094ca3

Please sign in to comment.