Skip to content

Commit

Permalink
Merge pull request #61 from aserto-dev/manifest-metadata
Browse files Browse the repository at this point in the history 
Support Aserto-Manifest-Request and If-Match/If-Not-Match
  • Loading branch information
@ronenh
ronenh authored Nov 17, 2023
2 parents 2c7a101 + 2efbb64 commit 2ab8a86
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 74 deletions.
15 changes: 8 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ go 1.20

require (
github.com/Masterminds/semver v1.5.0
github.com/aserto-dev/azm v0.0.16
github.com/aserto-dev/azm v0.0.17
github.com/aserto-dev/errors v0.0.6
github.com/aserto-dev/go-directory v0.30.4
github.com/aserto-dev/go-directory v0.30.5
github.com/bufbuild/protovalidate-go v0.4.1
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
github.com/gonvenience/ytbx v1.4.4
github.com/google/uuid v1.4.0
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
Expand Down Expand Up @@ -39,7 +40,7 @@ require (
github.com/gonvenience/text v1.0.7 // indirect
github.com/gonvenience/wrap v1.1.2 // indirect
github.com/google/cel-go v0.18.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
Expand All @@ -58,10 +59,10 @@ require (
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
33 changes: 17 additions & 16 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/aserto-dev/azm v0.0.16 h1:TqYF/tO29fakkXR0ZPUOXC6mJI9zO9DRsGKo4q1qlUc=
github.com/aserto-dev/azm v0.0.16/go.mod h1:oNNvPYysttO/otHryQDm6u71jAt/Qhs9izhrde7HlD0=
github.com/aserto-dev/azm v0.0.17 h1:jerSOk8WkNnmxt8eagT8mWSeOCswxpjQ4B1s2cZwass=
github.com/aserto-dev/azm v0.0.17/go.mod h1:oNNvPYysttO/otHryQDm6u71jAt/Qhs9izhrde7HlD0=
github.com/aserto-dev/errors v0.0.6 h1:iH5fkJwBGFPbcdS4B8mwvNdwODlhDEXXPduZtjLh6vo=
github.com/aserto-dev/errors v0.0.6/go.mod h1:kenI1gamsemaR2wS+M2un0kXIJ9exTrmeRT/fCFwlWc=
github.com/aserto-dev/go-directory v0.30.4 h1:MifQgaA2ino54mWcmJAbJZSmUPNqat0V/u60EpFd9Vs=
github.com/aserto-dev/go-directory v0.30.4/go.mod h1:jHn6ckERuci6nWz9vCzAbYoF2UnbOwEzl++sA1L3nyc=
github.com/aserto-dev/go-directory v0.30.5 h1:wOF1dtiqlNbfvb7iBH9qvgQ4e2jrIfMlRJW2ly1JhSI=
github.com/aserto-dev/go-directory v0.30.5/go.mod h1:qd/+uHrKvskPSN48FLGeZ/FoiORxjRmikCKRIp3pnYY=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/bufbuild/protovalidate-go v0.4.1 h1:ye/8S72WbEklCeltPkSEeT8Eu1A7P/gmMsmapkwqTFk=
github.com/bufbuild/protovalidate-go v0.4.1/go.mod h1:+p5FXfOjSEgLz5WBDTOMPMdQPXqALEERbJZU7huDCtA=
Expand All @@ -30,6 +30,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
Expand All @@ -38,7 +40,6 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
Expand All @@ -64,15 +65,15 @@ github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4=
github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
Expand Down Expand Up @@ -206,8 +207,8 @@ golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
Expand All @@ -228,12 +229,12 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA=
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI=
google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 h1:HJMDndgxest5n2y77fnErkM62iUsptE/H8p0dC2Huo4=
google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405/go.mod h1:oT32Z4o8Zv2xPQTg0pbVaPr0MPOH6f14RgXt7zfIpwg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U=
google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4=
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo=
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 h1:AB/lmRny7e2pLhFEYIbl5qkDAUt2h0ZRO4wGPhZf+ik=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.mod h1:67X1fPuzjcrkymZzZV1vvkFeTn2Rvc6lYF9MYFGCcwE=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
Expand Down
103 changes: 53 additions & 50 deletions pkg/directory/v3/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,19 @@ import (
dsm3 "github.com/aserto-dev/go-directory/aserto/directory/model/v3"
"github.com/aserto-dev/go-directory/pkg/derr"
"github.com/aserto-dev/go-directory/pkg/gateway/model/v3"
mnfst "github.com/aserto-dev/go-directory/pkg/manifest"
"github.com/aserto-dev/go-directory/pkg/pb"
"github.com/aserto-dev/go-edge-ds/pkg/bdb"
"github.com/aserto-dev/go-edge-ds/pkg/ds"
"github.com/go-http-utils/headers"
"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
"github.com/samber/lo"

"github.com/bufbuild/protovalidate-go"
"github.com/pkg/errors"
"github.com/rs/zerolog"
bolt "go.etcd.io/bbolt"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
"google.golang.org/protobuf/types/known/timestamppb"
)
Expand Down Expand Up @@ -59,16 +61,6 @@ func (s *Model) GetManifest(req *dsm3.GetManifestRequest, stream dsm3.Model_GetM
return derr.ErrProtoValidate.Msg(err.Error())
}

hdr, ok := metadata.FromIncomingContext(stream.Context())
if !ok {
return status.Errorf(codes.DataLoss, "failed to get metadata")
}
amr, ok := hdr["aserto-model-request"]
getModel := false
if ok && amr[0] == "model-only" {
getModel = true
}

md := &dsm3.Metadata{UpdatedAt: timestamppb.Now(), Etag: ""}

modelErr := s.store.DB().View(func(tx *bolt.Tx) error {
Expand All @@ -90,54 +82,60 @@ func (s *Model) GetManifest(req *dsm3.GetManifestRequest, stream dsm3.Model_GetM
return err
}

body := &dsm3.Body{}
inMD, _ := metadata.FromIncomingContext(stream.Context())
if lo.Contains(inMD.Get(headers.IfNoneMatch), md.Etag) {
return nil
}

amr := mnfst.IncomingManifestRequest(stream.Context())
if amr.WithBody() {
body := &dsm3.Body{}

for curByte := 0; curByte < len(manifest.Body.Data); curByte += model.MaxChunkSizeBytes {
if curByte+model.MaxChunkSizeBytes > len(manifest.Body.Data) {
body.Data = manifest.Body.Data[curByte:len(manifest.Body.Data)]
} else {
body.Data = manifest.Body.Data[curByte : curByte+model.MaxChunkSizeBytes]
}

if err := stream.Send(&dsm3.GetManifestResponse{
Msg: &dsm3.GetManifestResponse_Body{
Body: body,
},
}); err != nil {
return err
}
}
}

for curByte := 0; curByte < len(manifest.Body.Data); curByte += model.MaxChunkSizeBytes {
if curByte+model.MaxChunkSizeBytes > len(manifest.Body.Data) {
body.Data = manifest.Body.Data[curByte:len(manifest.Body.Data)]
} else {
body.Data = manifest.Body.Data[curByte : curByte+model.MaxChunkSizeBytes]
if amr.WithModel() {
model, err := ds.Manifest(md).GetModel(stream.Context(), tx)
switch {
case bdb.ErrIsNotFound(err):
return derr.ErrNotFound.Msg("model")
case err != nil:
return errors.Errorf("failed to get model")
}

m := pb.NewStruct()
r, err := model.Reader()
if err != nil {
return err
}

if err := pb.BufToProto(r, m); err != nil {
return err
}

if err := stream.Send(&dsm3.GetManifestResponse{
Msg: &dsm3.GetManifestResponse_Body{
Body: body,
Msg: &dsm3.GetManifestResponse_Model{
Model: m,
},
}); err != nil {
return err
}
}

if !getModel {
return nil
}

model, err := ds.Manifest(md).GetModel(stream.Context(), tx)
switch {
case bdb.ErrIsNotFound(err):
return derr.ErrNotFound.Msg("model")
case err != nil:
return errors.Errorf("failed to get model")
}

m := pb.NewStruct()
r, err := model.Reader()
if err != nil {
return err
}

if err := pb.BufToProto(r, m); err != nil {
return err
}

if err := stream.Send(&dsm3.GetManifestResponse{
Msg: &dsm3.GetManifestResponse_Model{
Model: m,
},
}); err != nil {
return err
}

return nil
})

Expand All @@ -148,6 +146,11 @@ func (s *Model) SetManifest(stream dsm3.Model_SetManifestServer) error {
logger := s.logger.With().Str("method", "SetManifest").Logger()
logger.Trace().Send()

etag := metautils.ExtractIncoming(stream.Context()).Get(headers.IfMatch)
if etag != "" && etag != s.store.MC().Metadata().ETag {
return derr.ErrHashMismatch
}

h := fnv.New64a()
h.Reset()

Expand Down
2 changes: 1 addition & 1 deletion pkg/ds/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func (i *object) Validate(mc *cache.Cache) (bool, error) {
}

if !mc.ObjectExists(model.ObjectName(i.Object.Type)) {
return false, derr.ErrObjectNotFound.Msg(i.Object.Type)
return false, derr.ErrObjectTypeNotFound.Msg(i.Object.Type)
}

return true, nil
Expand Down

0 comments on commit 2ab8a86

Please sign in to comment.