From 03b80a89367f0e9d312cca1aed2c2ccc04d62b0d Mon Sep 17 00:00:00 2001 From: Paul Norton Date: Tue, 19 Dec 2023 16:40:49 -0500 Subject: [PATCH] lift common policy code goal-eval --- Taskfile.yaml | 2 +- go.mod | 9 +- go.sum | 77 +++++++ internal/test_util/logger.go | 16 ++ internal/test_util/pointer.go | 7 + policy/evaluators/data/convert.go | 152 +++++++++++++ policy/evaluators/data/fixed.go | 28 +++ policy/evaluators/data/query.go | 197 +++++++++++++++++ policy/evaluators/data/source.go | 56 +++++ policy/evaluators/differ.go | 73 ++++++ policy/evaluators/evaluators.go | 23 ++ policy/goals/entities.go | 31 +++ policy/goals/entities_test.go | 44 ++++ policy/goals/types.go | 50 +++++ policy/graphql/cves_by_pkg.go | 42 ++++ policy/graphql/fake_client.go | 28 +++ policy/graphql/graphql.go | 87 ++++++++ policy/graphql/image_details_by_digest.go | 90 ++++++++ policy/graphql/pkgs_by_digest.go | 95 ++++++++ policy/graphql/queries.go | 100 +++++++++ policy/graphql/types.go | 82 +++++++ policy/handler.go | 258 ++++++++++++++++++++++ policy/query/async.go | 73 ++++++ policy/query/types.go | 27 +++ policy/skills/parse.go | 56 +++++ policy/skills/parse_test.go | 97 ++++++++ policy/storage/fs.go | 44 ++++ policy/storage/gcs.go | 75 +++++++ policy/storage/storage.go | 23 ++ 29 files changed, 1940 insertions(+), 2 deletions(-) create mode 100644 internal/test_util/logger.go create mode 100644 internal/test_util/pointer.go create mode 100644 policy/evaluators/data/convert.go create mode 100644 policy/evaluators/data/fixed.go create mode 100644 policy/evaluators/data/query.go create mode 100644 policy/evaluators/data/source.go create mode 100644 policy/evaluators/differ.go create mode 100644 policy/evaluators/evaluators.go create mode 100644 policy/goals/entities.go create mode 100644 policy/goals/entities_test.go create mode 100644 policy/goals/types.go create mode 100644 policy/graphql/cves_by_pkg.go create mode 100644 policy/graphql/fake_client.go create mode 100644 policy/graphql/graphql.go create mode 100644 policy/graphql/image_details_by_digest.go create mode 100644 policy/graphql/pkgs_by_digest.go create mode 100644 policy/graphql/queries.go create mode 100644 policy/graphql/types.go create mode 100644 policy/handler.go create mode 100644 policy/query/async.go create mode 100644 policy/query/types.go create mode 100644 policy/skills/parse.go create mode 100644 policy/skills/parse_test.go create mode 100644 policy/storage/fs.go create mode 100644 policy/storage/gcs.go create mode 100644 policy/storage/storage.go diff --git a/Taskfile.yaml b/Taskfile.yaml index 488c91b..8d81743 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -20,7 +20,7 @@ vars: tasks: go:test: cmds: - - go test -v + - go test -v ./... --count=1 go:build: cmds: diff --git a/go.mod b/go.mod index 680c62f..66f3d7a 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,13 @@ go 1.19 require ( cloud.google.com/go/logging v1.8.1 + cloud.google.com/go/storage v1.31.0 github.com/google/uuid v1.3.1 + github.com/hasura/go-graphql-client v0.9.3 + github.com/mitchellh/hashstructure/v2 v2.0.1 github.com/sirupsen/logrus v1.9.0 golang.org/x/oauth2 v0.13.0 + google.golang.org/api v0.147.0 olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 ) @@ -14,6 +18,7 @@ require ( cloud.google.com/go v0.110.8 // indirect cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.2 // indirect cloud.google.com/go/longrunning v0.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -21,17 +26,19 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/klauspost/compress v1.10.3 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - google.golang.org/api v0.147.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/grpc v1.58.2 // indirect google.golang.org/protobuf v1.31.0 // indirect + nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 3aa1a80..b202603 100644 --- a/go.sum +++ b/go.sum @@ -6,10 +6,13 @@ cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/logging v1.8.1 h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU= cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= +cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -21,6 +24,26 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -29,6 +52,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -46,17 +71,44 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v1.5.0 h1:fDqblo50TEpD0LY7RXk/LFVYEVqo3+tXMNMPSVXA1yc= +github.com/graph-gophers/graphql-go v1.5.0/go.mod h1:YtmJZDLbF1YYNrlNAuiO5zAStUWc3XZT07iGsVqe1Os= +github.com/graph-gophers/graphql-transport-ws v0.0.2 h1:DbmSkbIGzj8SvHei6n8Mh9eLQin8PtA8xY9eCzjRpvo= +github.com/graph-gophers/graphql-transport-ws v0.0.2/go.mod h1:5BVKvFzOd2BalVIBFfnfmHjpJi/MZ5rOj8G55mXvZ8g= +github.com/hasura/go-graphql-client v0.9.3 h1:Xi3fqa2t9q4nJ2jM2AU8nB6qeAoMpbcYDiOSBnNAN1E= +github.com/hasura/go-graphql-client v0.9.3/go.mod h1:AarJlxO1I59MPqU/TC7gQP0BMFgPEqUTt5LYPvykasw= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mitchellh/hashstructure/v2 v2.0.1 h1:L60q1+q7cXE4JeEJJKMnh2brFIe3rZxCihYAB61ypAY= +github.com/mitchellh/hashstructure/v2 v2.0.1/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -65,15 +117,24 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI= +go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -87,6 +148,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -100,21 +162,31 @@ golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.147.0 h1:Can3FaQo9LlVqxJCodNmeZW/ib3/qKAY3rFeXiHo5gc= google.golang.org/api v0.147.0/go.mod h1:pQ/9j83DcmPd/5C9e2nFOdjjNkDZ1G+zkbK2uvdkJMs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -151,10 +223,15 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= diff --git a/internal/test_util/logger.go b/internal/test_util/logger.go new file mode 100644 index 0000000..b867070 --- /dev/null +++ b/internal/test_util/logger.go @@ -0,0 +1,16 @@ +package test_util + +import "github.com/atomist-skills/go-skill" + +func CreateEmptyLogger() skill.Logger { + return skill.Logger{ + Debug: func(msg string) {}, + Debugf: func(format string, a ...any) {}, + Info: func(msg string) {}, + Infof: func(format string, a ...any) {}, + Warn: func(msg string) {}, + Warnf: func(format string, a ...any) {}, + Error: func(msg string) {}, + Errorf: func(format string, a ...any) {}, + } +} diff --git a/internal/test_util/pointer.go b/internal/test_util/pointer.go new file mode 100644 index 0000000..0a92187 --- /dev/null +++ b/internal/test_util/pointer.go @@ -0,0 +1,7 @@ +package test_util + +// Pointer is useful for making pointers of literals in test cases +// e.g. Pointer(3) or Pointer("string") +func Pointer[T any](some T) *T { + return &some +} diff --git a/policy/evaluators/data/convert.go b/policy/evaluators/data/convert.go new file mode 100644 index 0000000..f74a242 --- /dev/null +++ b/policy/evaluators/data/convert.go @@ -0,0 +1,152 @@ +package data + +import ( + "time" + + "github.com/atomist-skills/go-skill/policy/graphql" +) + +func getPurlsFromPackages(packages []MetadataPackage) []string { + purls := []string{} + for _, pkg := range packages { + purls = append(purls, pkg.Purl) + } + return purls +} + +func getPackagesByPurl(packages []graphql.VulnerabilitiesByPackage) map[string]graphql.VulnerabilitiesByPackage { + result := map[string]graphql.VulnerabilitiesByPackage{} + for _, pkg := range packages { + result[pkg.Purl] = pkg + } + + return result +} + +func convertGraphqlToPackages(imagePackages graphql.ImagePackagesByDigest) ([]Package, error) { + nonEmptyHistories := []graphql.ImageHistory{} + for _, history := range imagePackages.ImageHistories { + if !history.EmptyLayer { + nonEmptyHistories = append(nonEmptyHistories, history) + } + } + + pkgs := []Package{} + for _, p := range imagePackages.ImagePackages.Packages { + locations := []PackageLocation{} + for _, location := range p.PackageLocations { + layerOrdinal := -1 + for _, layer := range imagePackages.ImageLayers.Layers { + if location.DiffId == layer.DiffId { + layerOrdinal = layer.Ordinal + break + } + } + + historyOrdinal := -1 + if len(nonEmptyHistories) > 0 && layerOrdinal > -1 { + historyOrdinal = nonEmptyHistories[layerOrdinal].Ordinal + } + + locations = append(locations, PackageLocation{ + LayerOrdinal: historyOrdinal, + Path: location.Path, + }) + } + + var namespace string + if p.Package.Namespace == nil { + namespace = "" + } else { + namespace = *p.Package.Namespace + } + + vulnerabilities, err := convertToVulnerabilities(p.Package.Vulnerabilities) + if err != nil { + return nil, err + } + + pkgs = append(pkgs, Package{ + Purl: p.Package.Purl, + Licenses: p.Package.Licenses, + Name: p.Package.Name, + Namespace: namespace, + Version: p.Package.Version, + Type: p.Package.Type, + Locations: locations, + Vulnerabilities: vulnerabilities, + }) + } + + return pkgs, nil +} + +func convertMetadataPackagesToPackages(metadataPackages []MetadataPackage, vulnerabilitiesByPackage []graphql.VulnerabilitiesByPackage) ([]Package, error) { + pkgsByPurl := getPackagesByPurl(vulnerabilitiesByPackage) + + packages := []Package{} + for _, mPkg := range metadataPackages { + pkg := pkgsByPurl[mPkg.Purl] + vulnerabilities, err := convertToVulnerabilities(pkg.Vulnerabilities) + if err != nil { + return nil, err + } + + packages = append(packages, Package{ + Licenses: mPkg.Licenses, + Name: mPkg.Name, + Namespace: mPkg.Namespace, + Version: mPkg.Version, + Purl: mPkg.Purl, + Type: mPkg.Type, + Vulnerabilities: vulnerabilities, + }) + } + + return packages, nil +} + +func convertToVulnerabilities(vulnerabilities []graphql.Vulnerability) ([]Vulnerability, error) { + result := []Vulnerability{} + + for _, vulnerability := range vulnerabilities { + publishedAt, err := time.Parse(time.RFC3339, vulnerability.PublishedAt) + if err != nil { + return nil, err + } + + updatedAt, err := time.Parse(time.RFC3339, vulnerability.UpdatedAt) + if err != nil { + return nil, err + } + + vulnerabilityResult := Vulnerability{ + Cvss: Cvss{}, + PublishedAt: publishedAt, + Source: vulnerability.Source, + SourceID: vulnerability.SourceID, + UpdatedAt: updatedAt, + VulnerableRange: vulnerability.VulnerableRange, + } + + if vulnerability.Cvss.Score != nil { + vulnerabilityResult.Cvss.Score = *vulnerability.Cvss.Score + } + + if vulnerability.Cvss.Severity != nil { + vulnerabilityResult.Cvss.Severity = *vulnerability.Cvss.Severity + } + + if vulnerability.URL != nil { + vulnerabilityResult.URL = *vulnerability.URL + } + + if vulnerability.FixedBy != nil { + vulnerabilityResult.FixedBy = *vulnerability.FixedBy + } + + result = append(result, vulnerabilityResult) + } + + return result, nil +} diff --git a/policy/evaluators/data/fixed.go b/policy/evaluators/data/fixed.go new file mode 100644 index 0000000..1ff2313 --- /dev/null +++ b/policy/evaluators/data/fixed.go @@ -0,0 +1,28 @@ +package data + +import ( + "context" + + "github.com/atomist-skills/go-skill/policy/graphql" + "github.com/atomist-skills/go-skill/policy/query" +) + +// FixedDataSource is only used for tests +type FixedDataSource struct { + Packages map[string][]Package + ImageDetailsByDigest *graphql.ImageDetailsByDigest +} + +func (s FixedDataSource) GetPackages(ctx context.Context, digest string) (*GetPackagesResult, error) { + return &GetPackagesResult{ + AsyncQueryMade: false, + Result: s.Packages[digest], + }, nil +} + +func (s FixedDataSource) GetImageDetailsByDigest(ctx context.Context, digest string, platform query.ImagePlatform) (*GetImageDetailsByDigestResult, error) { + return &GetImageDetailsByDigestResult{ + AsyncQueryMade: false, + Result: s.ImageDetailsByDigest, + }, nil +} diff --git a/policy/evaluators/data/query.go b/policy/evaluators/data/query.go new file mode 100644 index 0000000..67b0b47 --- /dev/null +++ b/policy/evaluators/data/query.go @@ -0,0 +1,197 @@ +package data + +import ( + "context" + "fmt" + + "github.com/atomist-skills/go-skill/policy/graphql" + "github.com/atomist-skills/go-skill/policy/query" + + "github.com/atomist-skills/go-skill" + "olympos.io/encoding/edn" +) + +type QueryDataSource struct { + client graphql.GraphqlClient + asyncClient *query.AsyncQueryClient + hasAsyncResult bool + pkgWithVulnerabilityOverride map[string][]Package + metadataPackages map[string][]MetadataPackage + imageDetailsOverride map[string]*graphql.ImageDetailsByDigest +} + +type Option func(s *QueryDataSource) error + +type MetadataPackage struct { + Licenses []string `edn:"licenses,omitempty"` // only needed for the license policy evaluation + Name string `edn:"name"` + Namespace string `edn:"namespace"` + Version string `edn:"version"` + Purl string `edn:"purl"` + Type string `edn:"type"` +} + +func NewQueryDataSource(ctx context.Context, req skill.RequestContext, opt ...Option) (DataSource, error) { + c, err := graphql.NewGraphqlSkillClient(ctx, req) + if err != nil { + return nil, err + } + + ds := QueryDataSource{ + client: c, + } + for _, option := range opt { + err = option(&ds) + if err != nil { + return nil, err + } + } + + return ds, nil +} + +func WithAsyncClient(asyncClient query.AsyncQueryClient) Option { + return func(s *QueryDataSource) error { + s.asyncClient = &asyncClient + return nil + } +} + +func WithFixedPackageList(packageList map[string][]MetadataPackage) Option { + return func(s *QueryDataSource) error { + s.metadataPackages = packageList + return nil + } +} + +func WithAsyncQueryResult(name string, result map[edn.Keyword]edn.RawMessage) Option { + return func(s *QueryDataSource) error { + switch name { + case graphql.ImagePackagesAsyncQueryName: + packagesResponse, err := graphql.GetImagePackagesByDigestAsyncCallback(result) + if err != nil { + return err + } + + if packagesResponse != nil { + packages, err := convertGraphqlToPackages(*packagesResponse) + if err != nil { + return err + } + + s.pkgWithVulnerabilityOverride = map[string][]Package{ + packagesResponse.Digest: packages, + } + } + case graphql.ImageDetailsAsyncQueryName: + detailsResponse, err := graphql.GetImageDetailsByDigestAsyncCallback(result) + if err != nil { + return err + } + + if detailsResponse != nil { + s.imageDetailsOverride = map[string]*graphql.ImageDetailsByDigest{ + detailsResponse.Digest: detailsResponse, + } + } + } + + s.hasAsyncResult = true + return nil + } +} + +func (s QueryDataSource) canMakeAsyncRequest() bool { + return s.asyncClient != nil && !s.hasAsyncResult +} + +func (s QueryDataSource) GetPackages(ctx context.Context, digest string) (*GetPackagesResult, error) { + if s.pkgWithVulnerabilityOverride != nil { + return &GetPackagesResult{ + AsyncQueryMade: false, + Result: s.pkgWithVulnerabilityOverride[digest], + }, nil + } + + if s.metadataPackages != nil { + metadataPackages := s.metadataPackages[digest] + purls := getPurlsFromPackages(metadataPackages) + vulnerabilitiesByPackage, err := s.client.GetVulnerabilitiesByPackage(ctx, purls) + if err != nil { + return nil, err + } + + packages, err := convertMetadataPackagesToPackages(metadataPackages, vulnerabilitiesByPackage) + if err != nil { + return nil, err + } + + return &GetPackagesResult{ + AsyncQueryMade: false, + Result: packages, + }, nil + } + + if s.canMakeAsyncRequest() { + err := s.client.GetImagePackagesByDigestAsync(ctx, digest, *s.asyncClient) + if err != nil { + return nil, err + } + + return &GetPackagesResult{ + AsyncQueryMade: true, + Result: nil, + }, nil + } + + packagesResponse, err := s.client.GetImagePackagesByDigest(ctx, digest) + if err != nil { + return nil, err + } + + var packages []Package + if packagesResponse != nil { + packages, err = convertGraphqlToPackages(*packagesResponse) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("nil result received from imagePackagesByDigest query") + } + + return &GetPackagesResult{ + AsyncQueryMade: false, + Result: packages, + }, nil +} + +func (s QueryDataSource) GetImageDetailsByDigest(ctx context.Context, digest string, platform query.ImagePlatform) (*GetImageDetailsByDigestResult, error) { + if s.imageDetailsOverride != nil { + return &GetImageDetailsByDigestResult{ + AsyncQueryMade: false, + Result: s.imageDetailsOverride[digest], + }, nil + } + + if s.canMakeAsyncRequest() { + err := s.client.GetImageDetailsByDigestAsync(ctx, digest, platform, *s.asyncClient) + if err != nil { + return nil, err + } + + return &GetImageDetailsByDigestResult{ + AsyncQueryMade: true, + Result: nil, + }, nil + } + + detailsResponse, err := s.client.GetImageDetailsByDigest(ctx, digest, platform) + if err != nil { + return nil, err + } + + return &GetImageDetailsByDigestResult{ + AsyncQueryMade: false, + Result: detailsResponse, + }, nil +} diff --git a/policy/evaluators/data/source.go b/policy/evaluators/data/source.go new file mode 100644 index 0000000..fb3f40f --- /dev/null +++ b/policy/evaluators/data/source.go @@ -0,0 +1,56 @@ +package data + +import ( + "context" + "time" + + "github.com/atomist-skills/go-skill/policy/graphql" + "github.com/atomist-skills/go-skill/policy/query" +) + +type Package struct { + Licenses []string + Name string + Namespace string + Version string + Purl string + Type string + Locations []PackageLocation + Vulnerabilities []Vulnerability +} + +type PackageLocation struct { + LayerOrdinal int + Path string +} + +type Vulnerability struct { + Cvss Cvss + FixedBy string + PublishedAt time.Time + Source string + SourceID string + UpdatedAt time.Time + URL string + VulnerableRange string +} + +type Cvss struct { + Severity string + Score float32 +} + +type GetPackagesResult struct { + AsyncQueryMade bool + Result []Package +} + +type GetImageDetailsByDigestResult struct { + AsyncQueryMade bool + Result *graphql.ImageDetailsByDigest +} + +type DataSource interface { + GetPackages(ctx context.Context, digest string) (*GetPackagesResult, error) + GetImageDetailsByDigest(ctx context.Context, digest string, platform query.ImagePlatform) (*GetImageDetailsByDigestResult, error) +} diff --git a/policy/evaluators/differ.go b/policy/evaluators/differ.go new file mode 100644 index 0000000..85a3e47 --- /dev/null +++ b/policy/evaluators/differ.go @@ -0,0 +1,73 @@ +package evaluators + +import ( + "fmt" + + "github.com/atomist-skills/go-skill/policy/goals" + + "github.com/atomist-skills/go-skill" + "github.com/mitchellh/hashstructure/v2" +) + +// GoalResultsDiffer checks if the current query results differ from the previous ones. +// It returns the storage id for the current query results. +func GoalResultsDiffer(log skill.Logger, queryResults []goals.GoalEvaluationQueryResult, digest string, goal goals.Goal, previousStorageId string) (bool, string, error) { + log.Infof("Generating storage id for goal %s, image %s", goal.Definition, digest) + hash, err := hashstructure.Hash(queryResults, hashstructure.FormatV2, nil) + if err != nil { + return false, "", fmt.Errorf("failed to generate storage id for goal %s, image %s: %s", goal.Definition, digest, err) + } + + storageId := fmt.Sprint(hash) + + differ := storageId != previousStorageId + + if differ { + log.Infof("New storage id [%s] differs from previous [%s]", storageId, previousStorageId) + } else { + log.Infof("New storage id matches previous [%s]", storageId) + } + + return differ, storageId, nil +} + +func isRelevantParam(str string) bool { + irrelevantParams := []string{"definitionName", "displayName", "description"} + for _, v := range irrelevantParams { + if v == str { + return false + } + } + + return true +} + +// Returns the config hash for the current skill config +func GoalConfigsDiffer(log skill.Logger, config skill.Configuration, digest string, goal goals.Goal, previousConfigHash string) (bool, string, error) { + log.Debugf("Generating config hash for goal %s, image %s", goal.Definition, digest) + + params := config.Parameters + values := map[string]interface{}{} + for _, p := range params { + if isRelevantParam(p.Name) { + values[p.Name] = p.Value + } + } + + hash, err := hashstructure.Hash(values, hashstructure.FormatV2, nil) + if err != nil { + return false, "", fmt.Errorf("failed to generate config hash for goal %s, image %s: %s", goal.Definition, digest, err) + } + + configHash := fmt.Sprint(hash) + + differ := configHash != previousConfigHash + + if differ { + log.Infof("New config hash [%s] differs from previous [%s]", configHash, previousConfigHash) + } else { + log.Infof("New config hash matches previous [%s]", configHash) + } + + return differ, configHash, nil +} diff --git a/policy/evaluators/evaluators.go b/policy/evaluators/evaluators.go new file mode 100644 index 0000000..75fc1ba --- /dev/null +++ b/policy/evaluators/evaluators.go @@ -0,0 +1,23 @@ +package evaluators + +import ( + "context" + + "github.com/atomist-skills/go-skill/policy/goals" + "github.com/atomist-skills/go-skill/policy/query" + + "github.com/atomist-skills/go-skill" + "olympos.io/encoding/edn" +) + +type EvaluatorFlags uint8 + +const ( + EVAL_SKIP_LOCAL EvaluatorFlags = 1 << iota + OPT_DIGEST +) + +type GoalEvaluator interface { + GetFlags() EvaluatorFlags + EvaluateGoal(ctx context.Context, req skill.RequestContext, commonData query.CommonSubscriptionQueryResult, subscriptionResults [][]edn.RawMessage) ([]goals.GoalEvaluationQueryResult, error) +} diff --git a/policy/goals/entities.go b/policy/goals/entities.go new file mode 100644 index 0000000..ab93b00 --- /dev/null +++ b/policy/goals/entities.go @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Atomist, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package goals + +import "time" + +func CreateEntitiesFromResults(results []GoalEvaluationQueryResult, goalDefinition string, goalConfiguration string, image string, storageId string, configHash string, evaluationTs time.Time) GoalEvaluationResultEntity { + return GoalEvaluationResultEntity{ + Definition: goalDefinition, + Configuration: goalConfiguration, + Subject: DockerImageEntity{Digest: image}, + DeviationCount: len(results), + StorageId: storageId, + ConfigHash: configHash, + CreatedAt: evaluationTs, + } +} diff --git a/policy/goals/entities_test.go b/policy/goals/entities_test.go new file mode 100644 index 0000000..56de800 --- /dev/null +++ b/policy/goals/entities_test.go @@ -0,0 +1,44 @@ +/* + * Copyright © 2023 Atomist, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package goals + +import ( + "testing" + "time" + + "olympos.io/encoding/edn" +) + +func TestCreateEntitiesFromResult(t *testing.T) { + result := `[{:name "CVE-2023-2650", :details {:purl "pkg:alpine/openssl@3.1.0-r4?os_name=alpine&os_version=3.18", :cve "CVE-2023-2650", :severity "HIGH", :fixed-by "3.1.1-r0"} }]` + + resultModel := []GoalEvaluationQueryResult{} + + edn.Unmarshal([]byte(result), &resultModel) + + evaluationTs := time.Date(2023, 7, 10, 20, 1, 41, 0, time.UTC) + + entity := CreateEntitiesFromResults(resultModel, "test-definition", "test-configuration", "test-image", "storage-id", "config-hash", evaluationTs) + + if entity.Definition != "test-definition" || entity.Configuration != "test-configuration" || entity.StorageId != "storage-id" || entity.CreatedAt.Format("2006-01-02T15:04:05.000Z") != "2023-07-10T20:01:41.000Z" { + t.Errorf("metadata not set correctly") + } + + if entity.DeviationCount != 1 { + t.Errorf("incorrect number of deviations, expected %d, got %d", 1, entity.DeviationCount) + } +} diff --git a/policy/goals/types.go b/policy/goals/types.go new file mode 100644 index 0000000..f89a967 --- /dev/null +++ b/policy/goals/types.go @@ -0,0 +1,50 @@ +/* + * Copyright © 2023 Atomist, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package goals + +import ( + "time" + + "github.com/atomist-skills/go-skill" + "olympos.io/encoding/edn" +) + +type Goal struct { + Args map[string]interface{} + Definition string + Configuration string +} + +type GoalEvaluationQueryResult struct { + Details map[edn.Keyword]interface{} `edn:"details"` +} + +type DockerImageEntity struct { + skill.Entity `entity-type:"docker/image"` + Digest string `edn:"docker.image/digest"` +} + +type GoalEvaluationResultEntity struct { + skill.Entity `entity-type:"goal/result"` + Definition string `edn:"goal.definition/name"` + Configuration string `edn:"goal.configuration/name"` + Subject DockerImageEntity `edn:"goal.result/subject"` + DeviationCount int `edn:"goal.result/deviation-count"` + StorageId string `edn:"goal.result/storage-id"` + ConfigHash string `edn:"goal.result/config-hash"` + CreatedAt time.Time `edn:"goal.result/created-at"` +} diff --git a/policy/graphql/cves_by_pkg.go b/policy/graphql/cves_by_pkg.go new file mode 100644 index 0000000..02494ec --- /dev/null +++ b/policy/graphql/cves_by_pkg.go @@ -0,0 +1,42 @@ +package graphql + +import ( + "context" + "encoding/json" +) + +type VulnerabilitiesByPackageResponse struct { + VulnerabilitiesByPackage []VulnerabilitiesByPackage `json:"vulnerabilitiesByPackage"` +} + +type VulnerabilitiesByPackage struct { + Purl string `json:"purl"` + Vulnerabilities []Vulnerability `json:"vulnerabilities"` +} + +func (client *GraphqlSkillClient) GetVulnerabilitiesByPackage(ctx context.Context, purls []string) ([]VulnerabilitiesByPackage, error) { + log := client.RequestContext.Log + + variables := map[string]interface{}{ + "context": gqlContext(client), + "packageUrls": purls, + } + + log.Infof("Graphql endpoint: %s", client.RequestContext.Event.Urls.Graphql) + log.Infof("Executing query: %s", vulnerabilitiesByPackageQuery) + log.Infof("Query variables: %v", variables) + + res, err := client.GraphqlClient.ExecRaw(ctx, vulnerabilitiesByPackageQuery, variables) + if err != nil { + return nil, err + } + + log.Infof("GraphQL query response: %s", string(res)) + + var r VulnerabilitiesByPackageResponse + if err := json.Unmarshal(res, &r); err != nil { + return nil, err + } + + return r.VulnerabilitiesByPackage, nil +} diff --git a/policy/graphql/fake_client.go b/policy/graphql/fake_client.go new file mode 100644 index 0000000..65b3053 --- /dev/null +++ b/policy/graphql/fake_client.go @@ -0,0 +1,28 @@ +package graphql + +import ( + "context" + + "github.com/atomist-skills/go-skill" + "github.com/atomist-skills/go-skill/internal/test_util" + "github.com/atomist-skills/go-skill/policy/query" +) + +type FakeGraphqlClient struct { + ImageDetailsByDigest *ImageDetailsByDigest + ImagePackagesByDigest ImagePackagesByDigest +} + +func (client *FakeGraphqlClient) GetImageDetailsByDigest(ctx context.Context, digest string, platform query.ImagePlatform) (*ImageDetailsByDigest, error) { + return client.ImageDetailsByDigest, nil +} + +func (client *FakeGraphqlClient) GetImagePackagesByDigest(ctx context.Context, digest string) (ImagePackagesByDigest, error) { + return client.ImagePackagesByDigest, nil +} + +func (client *FakeGraphqlClient) GetRequestContext() skill.RequestContext { + return skill.RequestContext{ + Log: test_util.CreateEmptyLogger(), + } +} diff --git a/policy/graphql/graphql.go b/policy/graphql/graphql.go new file mode 100644 index 0000000..1be6e57 --- /dev/null +++ b/policy/graphql/graphql.go @@ -0,0 +1,87 @@ +package graphql + +import ( + "context" + "fmt" + "net/http" + + "github.com/atomist-skills/go-skill/policy/query" + + "github.com/atomist-skills/go-skill" + "github.com/hasura/go-graphql-client" + "golang.org/x/oauth2" + "olympos.io/encoding/edn" +) + +type GraphqlSkillClient struct { + Url string + GraphqlClient *graphql.Client + AsyncQueryClient query.AsyncQueryClient + RequestContext skill.RequestContext +} + +type AsyncGraphqlQuery struct { + Query string `edn:"query"` + Variables map[edn.Keyword]interface{} `edn:"variables"` +} + +type GraphqlClient interface { + GetVulnerabilitiesByPackage(ctx context.Context, purls []string) ([]VulnerabilitiesByPackage, error) + GetImageDetailsByDigest(ctx context.Context, digest string, platform query.ImagePlatform) (*ImageDetailsByDigest, error) + GetImagePackagesByDigest(ctx context.Context, digest string) (*ImagePackagesByDigest, error) + + GetImagePackagesByDigestAsync(ctx context.Context, digest string, asyncClient query.AsyncQueryClient) error + GetImageDetailsByDigestAsync(ctx context.Context, digest string, platform query.ImagePlatform, asyncClient query.AsyncQueryClient) error + + GetRequestContext() skill.RequestContext +} + +func (client *GraphqlSkillClient) GetRequestContext() skill.RequestContext { + return client.RequestContext +} + +func NewGraphqlSkillClient(ctx context.Context, req skill.RequestContext) (GraphqlClient, error) { + httpClient := oauth2.NewClient(ctx, oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: req.Event.Token, TokenType: "Bearer"}, + )) + + return &GraphqlSkillClient{ + Url: req.Event.Urls.Graphql, + GraphqlClient: graphql.NewClient(req.Event.Urls.Graphql, httpClient). + WithRequestModifier(func(r *http.Request) { + r.Header.Add("Accept", "application/json") + }), + RequestContext: req, + }, nil +} + +func gqlContext(client *GraphqlSkillClient) GqlContext { + return GqlContext{ + TeamId: client.RequestContext.Event.WorkspaceId, + Organization: client.RequestContext.Event.Organization, + } +} + +func decodeEdnResponse[P interface{}](result map[edn.Keyword]edn.RawMessage) (*P, error) { + type resultType struct { + Data P `edn:"data"` + Errors []struct { + Message string `edn:"message"` + } `edn:"errors"` + } + + ednboby, err := edn.Marshal(result) + if err != nil { + return nil, err + } + var decoded resultType + err = edn.Unmarshal(ednboby, &decoded) + if err != nil { + return nil, err + } + + if len(decoded.Errors) > 0 { + return nil, fmt.Errorf(decoded.Errors[0].Message) + } + return &decoded.Data, nil +} diff --git a/policy/graphql/image_details_by_digest.go b/policy/graphql/image_details_by_digest.go new file mode 100644 index 0000000..8253182 --- /dev/null +++ b/policy/graphql/image_details_by_digest.go @@ -0,0 +1,90 @@ +package graphql + +import ( + "context" + "encoding/json" + + "github.com/atomist-skills/go-skill/policy/query" + "olympos.io/encoding/edn" +) + +type ImageDetailsByDigestResponse struct { + ImageDetailsByDigest *ImageDetailsByDigest `json:"imageDetailsByDigest" edn:"imageDetailsByDigest"` +} + +type ImageDetailsByDigest struct { + Digest string `json:"digest" edn:"digest"` + BaseImage *BaseImage `json:"baseImage" edn:"baseImage"` + BaseImageTag *Tag `json:"baseImageTag" edn:"baseImageTag"` +} + +const ImageDetailsAsyncQueryName string = "async-query-image-details" + +// GetImageDetailsByDigest fetches and returns a list of base images used in the +// creation of the image specified in digest. +func (client *GraphqlSkillClient) GetImageDetailsByDigest(ctx context.Context, digest string, platform query.ImagePlatform) (*ImageDetailsByDigest, error) { + log := client.RequestContext.Log + + variables := map[string]interface{}{ + "context": gqlContext(client), + "digest": digest, + "platform": ImagePlatform{ + Architecture: platform.Architecture, + Os: platform.Os, + }, + } + + log.Infof("Executing query: %s with vars %v", baseImagesByDigest, variables) + + res, err := client.GraphqlClient.ExecRaw(ctx, baseImagesByDigest, variables) + if err != nil { + return nil, err + } + + log.Infof("GraphQL query response: %s", string(res)) + + var r ImageDetailsByDigestResponse + err = json.Unmarshal(res, &r) + if err != nil { + return nil, err + } + + return r.ImageDetailsByDigest, nil +} + +func (client *GraphqlSkillClient) GetImageDetailsByDigestAsync(ctx context.Context, digest string, platform query.ImagePlatform, asyncClient query.AsyncQueryClient) error { + log := client.RequestContext.Log + + variables := map[edn.Keyword]interface{}{ + "context": gqlContext(client), + "digest": digest, + "platform": ImagePlatform{ + Architecture: platform.Architecture, + Os: platform.Os, + }, + } + + log.Infof("Executing query: %s with vars %v", baseImagesByDigest, variables) + + log.Infof("Async query") + + query := AsyncGraphqlQuery{ + Query: baseImagesByDigest, + Variables: variables, + } + + return asyncClient.SubmitAsyncQuery(ImageDetailsAsyncQueryName, client.Url, query) +} + +func GetImageDetailsByDigestAsyncCallback(response map[edn.Keyword]edn.RawMessage) (*ImageDetailsByDigest, error) { + result, err := decodeEdnResponse[ImageDetailsByDigestResponse](response) + if err != nil { + return nil, err + } + + if result.ImageDetailsByDigest == nil { + return nil, nil + } + + return (*result).ImageDetailsByDigest, nil +} diff --git a/policy/graphql/pkgs_by_digest.go b/policy/graphql/pkgs_by_digest.go new file mode 100644 index 0000000..8da5b4e --- /dev/null +++ b/policy/graphql/pkgs_by_digest.go @@ -0,0 +1,95 @@ +package graphql + +import ( + "context" + "encoding/json" + + "github.com/atomist-skills/go-skill/policy/query" + + "olympos.io/encoding/edn" +) + +type ImagePackagesByDigestResponse struct { + ImagePackagesByDigest *ImagePackagesByDigest `json:"imagePackagesByDigest" edn:"imagePackagesByDigest"` +} + +type ImagePackagesByDigest struct { + Digest string `json:"digest" edn:"digest"` + ImagePackages ImagePackages `json:"imagePackages" edn:"imagePackages"` + ImageHistories []ImageHistory `json:"imageHistories" edn:"imageHistories"` + ImageLayers ImageLayers `json:"imageLayers" edn:"imageLayers"` +} + +const ImagePackagesAsyncQueryName string = "async-query-packages" + +func (client *GraphqlSkillClient) GetImagePackagesByDigest(ctx context.Context, digest string) (*ImagePackagesByDigest, error) { + log := client.RequestContext.Log + + variables := map[string]interface{}{ + "context": gqlContext(client), + "digest": digest, + } + + log.Infof("Graphql endpoint: %s", client.RequestContext.Event.Urls.Graphql) + log.Infof("Executing query: %s", imagePackagesByDigestQuery) + log.Infof("Query variables: %v", variables) + + res, err := client.GraphqlClient.ExecRaw(ctx, imagePackagesByDigestQuery, variables) + if err != nil { + return nil, err + } + + log.Infof("GraphQL query response: %s", string(res)) + + imagePackages, err := getPackagesFromJsonResponse(res) + if err != nil { + return nil, err + } + + if imagePackages != nil { + log.Infof("Found %d packages for digest %s", len(imagePackages.ImagePackages.Packages), digest) + } else { + log.Infof("Empty package response for digest %s", digest) + } + + return imagePackages, nil +} + +func (client *GraphqlSkillClient) GetImagePackagesByDigestAsync(ctx context.Context, digest string, asyncClient query.AsyncQueryClient) error { + log := client.RequestContext.Log + + variables := map[edn.Keyword]interface{}{ + "context": gqlContext(client), + "digest": digest, + } + + log.Infof("Graphql endpoint: %s", client.RequestContext.Event.Urls.Graphql) + log.Infof("Executing query: %s", imagePackagesByDigestQuery) + log.Infof("Query variables: %v", variables) + log.Infof("Async query") + + query := AsyncGraphqlQuery{ + Query: imagePackagesByDigestQuery, + Variables: variables, + } + + return asyncClient.SubmitAsyncQuery(ImagePackagesAsyncQueryName, client.Url, query) +} + +func GetImagePackagesByDigestAsyncCallback(response map[edn.Keyword]edn.RawMessage) (*ImagePackagesByDigest, error) { + result, err := decodeEdnResponse[ImagePackagesByDigestResponse](response) + if err != nil { + return nil, err + } + + return (*result).ImagePackagesByDigest, nil +} + +func getPackagesFromJsonResponse(b []byte) (*ImagePackagesByDigest, error) { + var r ImagePackagesByDigestResponse + if err := json.Unmarshal(b, &r); err != nil { + return nil, err + } + + return r.ImagePackagesByDigest, nil +} diff --git a/policy/graphql/queries.go b/policy/graphql/queries.go new file mode 100644 index 0000000..f1f0031 --- /dev/null +++ b/policy/graphql/queries.go @@ -0,0 +1,100 @@ +package graphql + +const ( + // language=graphql + vulnerabilitiesByPackageQuery = ` + query ($context: Context!, $packageUrls: [String!]!) { + vulnerabilitiesByPackage(context: $context, packageUrls: $packageUrls) { + purl + vulnerabilities { + cvss { + severity + score + } + fixedBy + publishedAt + source + sourceId + updatedAt + url + vulnerableRange + } + } + }` + + // language=graphql + imagePackagesByDigestQuery = ` + query ($context: Context!, $digest: String!) { + imagePackagesByDigest(context: $context, digest: $digest) { + digest + imagePackages { + packages { + locations { + diffId + path + } + package { + licenses + name + namespace + version + purl + type + vulnerabilities { + cvss { + severity + score + } + fixedBy + publishedAt + source + sourceId + updatedAt + url + vulnerableRange + } + } + } + } + imageHistories { + emptyLayer + ordinal + } + imageLayers { + layers { + diffId + ordinal + } + } + } + } +` + + // language=graphql + baseImagesByDigest = ` + query ($context: Context!, $digest: String!, $platform: ImagePlatform!) { + imageDetailsByDigest( + context: $context + digest: $digest + platform: $platform + ) { + digest + baseImage { + digest + repository { + hostName + repoName + } + tags { + name + current + } + } + baseImageTag { + name + current + } + } + } +` +) diff --git a/policy/graphql/types.go b/policy/graphql/types.go new file mode 100644 index 0000000..b77f802 --- /dev/null +++ b/policy/graphql/types.go @@ -0,0 +1,82 @@ +package graphql + +type GqlContext struct { + TeamId string `json:"teamId"` + Organization string `json:"organization"` +} + +type ImagePlatform struct { + Architecture string `json:"architecture"` + Os string `json:"os"` + Variant string `json:"variant,omitempty"` +} + +type BaseImage struct { + Digest string `json:"digest" edn:"digest"` + Repository Repository `json:"repository" edn:"repository"` + Tags []Tag `json:"tags" edn:"tags"` +} + +type Repository struct { + HostName string `json:"hostName" edn:"hostName"` + RepoName string `json:"repoName" edn:"repoName"` +} + +type Tag struct { + Name string `json:"name" edn:"name"` + Current bool `json:"current" edn:"current"` +} + +type ImagePackages struct { + Packages []Packages `json:"packages" edn:"packages"` +} + +type ImageHistory struct { + EmptyLayer bool `json:"emptyLayer" edn:"emptyLayer"` + Ordinal int `json:"ordinal" edn:"ordinal"` +} + +type ImageLayers struct { + Layers []ImageLayer `json:"layers" edn:"layers"` +} + +type ImageLayer struct { + DiffId string `json:"diffId" edn:"diffId"` + Ordinal int `json:"ordinal" edn:"ordinal"` +} + +type Packages struct { + Package PackageWithLicenses `json:"package" edn:"package"` + PackageLocations []PackageLocation `json:"locations" edn:"locations"` +} + +type PackageWithLicenses struct { + Licenses []string `json:"licenses" edn:"licenses"` + Name string `json:"name" edn:"name"` + Namespace *string `json:"namespace" edn:"namespace"` + Version string `json:"version" edn:"version"` + Purl string `json:"purl" edn:"purl"` + Type string `json:"type" edn:"type"` + Vulnerabilities []Vulnerability `json:"vulnerabilities" edn:"vulnerabilities"` +} + +type PackageLocation struct { + DiffId string `json:"diffId" edn:"diffId"` + Path string `json:"path" edn:"path"` +} + +type Vulnerability struct { + Cvss Cvss `json:"cvss"` + FixedBy *string `json:"fixedBy"` + PublishedAt string `json:"publishedAt"` + Source string `json:"source"` + SourceID string `json:"sourceId"` + UpdatedAt string `json:"updatedAt"` + URL *string `json:"url"` + VulnerableRange string `json:"vulnerableRange"` +} + +type Cvss struct { + Severity *string `json:"severity"` + Score *float32 `json:"score"` +} diff --git a/policy/handler.go b/policy/handler.go new file mode 100644 index 0000000..af30b74 --- /dev/null +++ b/policy/handler.go @@ -0,0 +1,258 @@ +package policy + +import ( + "context" + b64 "encoding/base64" + "fmt" + "time" + + "github.com/atomist-skills/go-skill" + "github.com/atomist-skills/go-skill/policy/evaluators" + "github.com/atomist-skills/go-skill/policy/evaluators/data" + "github.com/atomist-skills/go-skill/policy/goals" + "github.com/atomist-skills/go-skill/policy/query" + "github.com/atomist-skills/go-skill/policy/storage" + "github.com/atomist-skills/go-skill/util" + "olympos.io/encoding/edn" +) + +type EvaluatorSelector func(ctx context.Context, req skill.RequestContext, goal *goals.Goal, dataSource data.DataSource) (evaluators.GoalEvaluator, error) + +var evaluatorSelector EvaluatorSelector + +func StartPolicy( + subscriptionNames []string, + selector EvaluatorSelector, +) { + evaluatorSelector = selector + handlers := skill.Handlers{ + "async-query-packages": evaluateGoalsWithAsyncQueryResult, + "async-query-image-details": evaluateGoalsWithAsyncQueryResult, + + "evaluate_goals_locally": evaluateGoalsLocally, + } + + for _, n := range subscriptionNames { + handlers[n] = evaluateGoals + } + + skill.Start(handlers) +} + +// EvaluateGoalsLocally runs the goal evaluation locally and returns the results without transacting them. +func evaluateGoalsLocally(ctx context.Context, req skill.RequestContext) skill.Status { + goalName := req.Event.Skill.Name + + cfg := req.Event.Context.SyncRequest.Configuration.Name + params := req.Event.Context.SyncRequest.Configuration.Parameters + + values := map[string]interface{}{} + for _, p := range params { + values[p.Name] = p.Value + } + + if _, ok := values["definitionName"]; !ok { + return skill.NewFailedStatus("Missing definition name in policy skill configuration") + } + + goal := goals.Goal{ + Definition: values["definitionName"].(string), + Configuration: cfg, + Args: values, + } + + metaPkgs := util.Decode[[]data.MetadataPackage](req.Event.Context.SyncRequest.Metadata["packages"]) + + digest := "localDigest" + + subscriptionResult := query.CommonSubscriptionQueryResult{ + ImageDigest: digest, + } + + dataSource, err := data.NewQueryDataSource(ctx, req, + data.WithFixedPackageList(map[string][]data.MetadataPackage{ + digest: metaPkgs, + }), + ) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("unable to create data source: %s", err.Error())) + } + + evaluator, err := evaluatorSelector(ctx, req, &goal, dataSource) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("unable to create evaluator: %s", err.Error())) + } + + if evaluator.GetFlags()&evaluators.EVAL_SKIP_LOCAL != 0 { + return skill.NewCompletedStatus("Skipped eval due to EVAL_SKIP_LOCAL") + } + + results, err := evaluator.EvaluateGoal(ctx, req, subscriptionResult, [][]edn.RawMessage{}) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Error evaluating goal %s", err.Error())) + } + + return skill.Status{ + State: skill.Completed, + Reason: fmt.Sprintf("Goal %s evaluated", goalName), + SyncRequest: results, + } +} + +func evaluateGoalsWithAsyncQueryResult(ctx context.Context, req skill.RequestContext) skill.Status { + asyncClient := query.NewAsyncQueryClient(req.Log, req.Event.Token, req.Event.Context.AsyncQueryResult.Metadata) + + metadata := req.Event.Context.AsyncQueryResult.Metadata + encoded, err := b64.StdEncoding.DecodeString(metadata) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to decode async metadata [%s]", err.Error())) + } + var subscriptionResult [][]edn.RawMessage + + err = edn.Unmarshal(encoded, &subscriptionResult) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to unmarshal async metadata [%s]", err.Error())) + } + + dataSource, err := data.NewQueryDataSource(ctx, req, + data.WithAsyncQueryResult(req.Event.Context.AsyncQueryResult.Name, req.Event.Context.AsyncQueryResult.Result), + data.WithAsyncClient(asyncClient)) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to create data source [%s]", err.Error())) + } + + return evaluateGoalWithData(ctx, req, dataSource, subscriptionResult, req.Event.Context.AsyncQueryResult.Configuration) +} + +// EvaluateGoals runs the goal evaluation and returns the results after transacting them. +func evaluateGoals(ctx context.Context, req skill.RequestContext) skill.Status { + edn, err := edn.Marshal(req.Event.Context.Subscription.Result) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to marshal metadata [%s]", err.Error())) + } + + encodedMetadata := b64.StdEncoding.EncodeToString(edn) + + asyncClient := query.NewAsyncQueryClient(req.Log, req.Event.Token, encodedMetadata) + + dataSource, err := data.NewQueryDataSource(ctx, req, + data.WithAsyncClient(asyncClient)) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to create data source [%s]", err.Error())) + } + + return evaluateGoalWithData(ctx, req, dataSource, req.Event.Context.Subscription.Result, req.Event.Context.Subscription.Configuration) +} + +func evaluateGoalWithData(ctx context.Context, req skill.RequestContext, dataSource data.DataSource, subscriptionResult [][]edn.RawMessage, configuration skill.Configuration) skill.Status { + goalName := req.Event.Skill.Name + + cfg := configuration.Name + params := configuration.Parameters + + values := map[string]interface{}{} + for _, p := range params { + values[p.Name] = p.Value + } + + if _, ok := values["definitionName"]; !ok { + return skill.NewFailedStatus("Missing definition name in policy skill configuration") + } + + goal := goals.Goal{ + Definition: values["definitionName"].(string), + Configuration: cfg, + Args: values, + } + + evaluator, err := evaluatorSelector(ctx, req, &goal, dataSource) + if err != nil { + req.Log.Errorf(err.Error()) + return skill.NewFailedStatus(fmt.Sprintf("Failed to create goal evaluator: %s", err.Error())) + } + + es, err := storage.NewEvaluationStorage(ctx) + if err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to create evaluation storage: %s", err.Error())) + } + + var entities []interface{} + var commonResults query.CommonSubscriptionQueryResult + var previousStorageId string + var previousConfigHash string + + // Find correct storageId and configHash from subscription result by only returning n/a if that is the only option + for _, result := range subscriptionResult { + storageTuple := util.Decode[[]string](result[1]) + if previousStorageId == "" || previousStorageId == "n/a" { + previousStorageId = storageTuple[0] + } + if previousConfigHash == "" || previousConfigHash == "n/a" { + previousConfigHash = storageTuple[1] + } + } + + subscriptionResults := [][]edn.RawMessage{} + for _, result := range subscriptionResult { + // Filter out duplicate results if we have real storage id and n/a + storageTuple := util.Decode[[]string](result[1]) + resultPreviousStorageId := storageTuple[0] + if resultPreviousStorageId == previousStorageId { + subscriptionResults = append(subscriptionResults, result) + } + + commonResults = util.Decode[query.CommonSubscriptionQueryResult](result[0]) + } + + digest := commonResults.ImageDigest + req.Log.Infof("Evaluating goal %s for digest %s ", goalName, digest) + evaluationTs := time.Now().UTC() + + qr, err := evaluator.EvaluateGoal(ctx, req, commonResults, subscriptionResults) + if err != nil { + req.Log.Errorf("Failed to evaluate goal %s for digest %s: %s", goal.Definition, digest, err.Error()) + return skill.NewFailedStatus("Failed to evaluate goal") + } + + if qr == nil { + req.Log.Infof("goal %s returned no data for digest %s, skipping storing results", goal.Definition, digest) + return skill.NewCompletedStatus(fmt.Sprintf("Goal %s evaluated - no data found", goalName)) + } + + configDiffer, configHash, err := evaluators.GoalConfigsDiffer(req.Log, configuration, digest, goal, previousConfigHash) + if err != nil { + req.Log.Errorf("Failed to check if config hash changed for digest: %s", digest, err) + req.Log.Warnf("Will continue with the evaluation nonetheless") + configDiffer = true + } + + differ, storageId, err := evaluators.GoalResultsDiffer(req.Log, qr, digest, goal, previousStorageId) + if err != nil { + req.Log.Errorf("Failed to check if goal results changed for digest: %s", digest, err) + req.Log.Warnf("Will continue with the evaluation nonetheless") + differ = true + } + + if differ { + if err := es.Store(ctx, qr, storageId, req.Event.Environment, req.Log); err != nil { + return skill.NewFailedStatus(fmt.Sprintf("Failed to store evaluation results for digest %s: %s", digest, err.Error())) + } + } + + if differ || configDiffer { + entity := goals.CreateEntitiesFromResults(qr, goal.Definition, goal.Configuration, digest, storageId, configHash, evaluationTs) + entities = append(entities, entity) + } + + if len(entities) > 0 { + err = req.NewTransaction().AddEntities(entities...).Transact() + if err != nil { + req.Log.Errorf(err.Error()) + } + req.Log.Info("Goal results transacted") + } else { + req.Log.Info("No goal results to transact") + } + + return skill.NewCompletedStatus(fmt.Sprintf("Goal %s evaluated", goalName)) +} diff --git a/policy/query/async.go b/policy/query/async.go new file mode 100644 index 0000000..50eb821 --- /dev/null +++ b/policy/query/async.go @@ -0,0 +1,73 @@ +package query + +import ( + "bytes" + "fmt" + "io" + "net/http" + "strings" + + "github.com/atomist-skills/go-skill" + "olympos.io/encoding/edn" +) + +type AsyncQueryRequest struct { + Name string `edn:"name"` + Body interface{} `edn:"body"` + Metadata string `edn:"metadata"` +} + +type AsyncQueryClient struct { + log skill.Logger + token string + metadata string +} + +func NewAsyncQueryClient(log skill.Logger, token string, metadata string) AsyncQueryClient { + return AsyncQueryClient{ + log: log, + token: token, + metadata: metadata, + } +} + +func (c *AsyncQueryClient) SubmitAsyncQuery(name string, url string, body interface{}) error { + request := AsyncQueryRequest{ + Name: name, + Body: body, + Metadata: c.metadata, + } + + edn, err := edn.Marshal(request) + if err != nil { + return err + } + + c.log.Infof("Async request: %s", string(edn)) + + url = url + ":enqueue" + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(edn)) + if err != nil { + return err + } + + req.Header.Add("Content-Type", "application/edn") + + authToken := c.token + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken)) + + r, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + if r.StatusCode >= 400 { + buf := new(strings.Builder) + _, _ = io.Copy(buf, r.Body) + body := buf.String() + + return fmt.Errorf("async request returned unexpected status %s: %s", r.Status, body) + } + + return nil +} diff --git a/policy/query/types.go b/policy/query/types.go new file mode 100644 index 0000000..51a4fd2 --- /dev/null +++ b/policy/query/types.go @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Atomist, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package query + +type ImagePlatform struct { + Architecture string `edn:"docker.platform/architecture" json:"architecture"` + Os string `edn:"docker.platform/os" json:"os"` +} + +type CommonSubscriptionQueryResult struct { + ImageDigest string `edn:"docker.image/digest"` + ImagePlatforms []ImagePlatform `edn:"docker.image/platform" json:"platforms"` +} diff --git a/policy/skills/parse.go b/policy/skills/parse.go new file mode 100644 index 0000000..1a2660d --- /dev/null +++ b/policy/skills/parse.go @@ -0,0 +1,56 @@ +package skills + +import "fmt" + +// ParseMultiChoiceArg parses the multi-choice skill parameter into a string array. +func ParseMultiChoiceArg(arg interface{}) []string { + var parsedArgs []string + + if arg == nil { + return parsedArgs + } + + if array, ok := arg.([]interface{}); ok { + for _, arg := range array { + parsedArgs = append(parsedArgs, fmt.Sprintf("%v", arg)) + } + } + + return parsedArgs +} + +// ParseStringArrayArgs parses the string-array skill parameter into a string array. +func ParseStringArrayArg(arg interface{}) []string { + var parsedArgs []string + + if arg == nil { + return parsedArgs + } + + if array, ok := arg.([]interface{}); ok { + for _, arg := range array { + parsedArgs = append(parsedArgs, fmt.Sprintf("%v", arg)) + } + } + + return parsedArgs +} + +// ParseIntArg parses the int skill parameter into an int64. +func ParseIntArg(arg interface{}) int64 { + if arg == nil { + return 0 + } + + parsedArgAsInt64, ok := arg.(int64) + if ok { + return parsedArgAsInt64 + } + + parsedArgAsFloat, ok := arg.(float64) + if ok { + return int64(parsedArgAsFloat) + } + + return 0 +} diff --git a/policy/skills/parse_test.go b/policy/skills/parse_test.go new file mode 100644 index 0000000..712d3a7 --- /dev/null +++ b/policy/skills/parse_test.go @@ -0,0 +1,97 @@ +package skills + +import ( + "testing" +) + +func TestParseMultiChoiceArg(t *testing.T) { + tests := []struct { + name string + arg interface{} + want []string + }{ + { + name: "Parse nil arg", + arg: nil, + want: []string{}, + }, + { + name: "Parse multiple args", + arg: []interface{}{"CRITICAL", "HIGH"}, + want: []string{"CRITICAL", "HIGH"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseMultiChoiceArg(tt.arg) + if len(got) != len(tt.want) { + t.Errorf("TestParseMultiChoiceArg() = %v, want %v", got, tt.arg) + } + + for i, v := range got { + if v != tt.want[i] { + t.Errorf("TestParseMultiChoiceArg() = %v, want %v", got, tt.arg) + } + } + }) + } +} + +func TestParseStringArrayArg(t *testing.T) { + tests := []struct { + name string + arg interface{} + want []string + }{ + { + name: "Parse nil arg", + arg: nil, + want: []string{}, + }, + { + name: "Parse multiple args", + arg: []interface{}{"MIT", "GPL-3.0"}, + want: []string{"MIT", "GPL-3.0"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseStringArrayArg(tt.arg) + if len(got) != len(tt.want) { + t.Errorf("ParseStringArrayArgs() = %v, want %v", got, tt.arg) + } + + for i, v := range got { + if v != tt.want[i] { + t.Errorf("ParseStringArrayArgs() = %v, want %v", got, tt.arg) + } + } + }) + } +} + +func TestParseIntArgs(t *testing.T) { + tests := []struct { + name string + arg interface{} + want int64 + }{ + { + name: "Parse nil arg", + arg: nil, + want: 0, + }, + { + name: "Parse single arg", + arg: int64(30), + want: 30, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ParseIntArg(tt.arg); got != tt.want { + t.Errorf("ParseIntArgs() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/policy/storage/fs.go b/policy/storage/fs.go new file mode 100644 index 0000000..3ffdc69 --- /dev/null +++ b/policy/storage/fs.go @@ -0,0 +1,44 @@ +package storage + +import ( + "context" + "os" + + "github.com/atomist-skills/go-skill/policy/goals" + + "github.com/atomist-skills/go-skill" + "olympos.io/encoding/edn" +) + +type FsStorage struct { + path string +} + +func NewFsStorage(ctx context.Context) (EvaluationStorage, error) { + return &FsStorage{ + path: os.TempDir(), + }, nil +} + +func (f *FsStorage) Store(ctx context.Context, results []goals.GoalEvaluationQueryResult, storageId, environment string, log skill.Logger) error { + log.Infof("Storing %d results", len(results)) + + content, err := edn.Marshal(results) + if err != nil { + return err + } + log.Infof("Content to store: %s", string(content)) + + file, err := os.Create(f.path + "/results.edn") + if err != nil { + return err + } + defer file.Close() + + _, err = file.WriteString(string(content)) + if err != nil { + return err + } + + return nil +} diff --git a/policy/storage/gcs.go b/policy/storage/gcs.go new file mode 100644 index 0000000..e04acc1 --- /dev/null +++ b/policy/storage/gcs.go @@ -0,0 +1,75 @@ +package storage + +import ( + "context" + "net/http" + + "github.com/atomist-skills/go-skill/policy/goals" + + "cloud.google.com/go/storage" + "github.com/atomist-skills/go-skill" + "google.golang.org/api/googleapi" + "olympos.io/encoding/edn" +) + +const ( + bucketName = "atm-policy-evaluation-results" +) + +type GcsStorage struct { + client *storage.Client + bucketName string + environment string +} + +func NewGcsStorage(ctx context.Context) (*GcsStorage, error) { + storageClient, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + + return &GcsStorage{ + client: storageClient, + bucketName: bucketName, + }, nil +} + +func (gcs *GcsStorage) Store(ctx context.Context, results []goals.GoalEvaluationQueryResult, storageId, environment string, log skill.Logger) error { + log.Infof("Storing %d results", len(results)) + + content, err := edn.Marshal(results) + if err != nil { + return err + } + log.Infof("Content to store: %s", string(content)) + + environmentBucketName := gcs.bucketName + + if gcs.environment != "" { + environmentBucketName = gcs.environment + "-" + gcs.environment + } + + bucket := gcs.client.Bucket(environmentBucketName) + storageObject := bucket.Object(storageId) + + w := storageObject.If(storage.Conditions{DoesNotExist: true}).NewWriter(ctx) + + _, err = w.Write(content) + if err != nil { + return err + } + + if err := w.Close(); err != nil { + switch e := err.(type) { + case *googleapi.Error: + // ignore if object already exists + if e.Code != http.StatusPreconditionFailed { + return err + } + default: + return err + } + } + + return nil +} diff --git a/policy/storage/storage.go b/policy/storage/storage.go new file mode 100644 index 0000000..cd1011b --- /dev/null +++ b/policy/storage/storage.go @@ -0,0 +1,23 @@ +package storage + +import ( + "context" + "os" + + "github.com/atomist-skills/go-skill/policy/goals" + + "github.com/atomist-skills/go-skill" +) + +type EvaluationStorage interface { + Store(ctx context.Context, results []goals.GoalEvaluationQueryResult, storageId, environment string, log skill.Logger) error +} + +// NewEvaluationStorage creates a new EvaluationStorage object based on the LOCAL_DEBUG environment variable. +func NewEvaluationStorage(ctx context.Context) (EvaluationStorage, error) { + if os.Getenv("LOCAL_DEBUG") == "true" { + return NewFsStorage(ctx) + } + + return NewGcsStorage(ctx) +}