From 9bcdabdad246c1da1da8db53b982d14f4f1a0cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Wed, 17 Apr 2024 20:21:09 +0200 Subject: [PATCH 1/8] tests: add helm upgrade tests --- go.mod | 31 ++++++ go.sum | 92 +++++++++++++++- modules/cli/cli_test.go | 2 +- test/e2e/environment.go | 74 +++++++++---- test/e2e/test_helm_install_upgrade.go | 146 ++++++++++++++++++++++++++ test/e2e/test_operator_logs.go | 2 +- test/e2e/test_upgrade.go | 4 +- test/e2e/test_webhook.go | 2 +- test/helpers/setup.go | 8 +- 9 files changed, 331 insertions(+), 30 deletions(-) create mode 100644 test/e2e/test_helm_install_upgrade.go diff --git a/go.mod b/go.mod index 4cb0cc4e6..29e057dea 100644 --- a/go.mod +++ b/go.mod @@ -31,26 +31,56 @@ require ( ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect github.com/avast/retry-go/v4 v4.5.1 // indirect + github.com/aws/aws-sdk-go v1.49.13 // indirect github.com/bombsimon/logrusr/v3 v3.1.0 // indirect + github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sql-driver/mysql v1.7.1 // indirect + github.com/gonvenience/bunt v1.3.5 // indirect + github.com/gonvenience/neat v1.3.12 // indirect + github.com/gonvenience/term v1.0.2 // indirect + github.com/gonvenience/text v1.0.7 // indirect + github.com/gonvenience/wrap v1.1.2 // indirect + github.com/gonvenience/ytbx v1.4.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-github/v48 v48.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/gruntwork-io/go-commons v0.8.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/homeport/dyff v1.6.0 // indirect + github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect github.com/kong/go-kong v0.54.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-ps v1.0.0 // indirect + github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pquerna/otp v1.2.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect + github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect + github.com/urfave/cli v1.22.14 // indirect + github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect @@ -119,6 +149,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 github.com/google/gofuzz v1.2.0 // indirect + github.com/gruntwork-io/terratest v0.46.13 github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index d6e3df8f8..62f61087a 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= @@ -22,12 +24,17 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= +github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= +github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ= github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -44,7 +51,11 @@ github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZV github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,6 +80,7 @@ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -77,6 +89,9 @@ github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -94,6 +109,8 @@ github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHa github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-task/slim-sprig v2.20.0+incompatible h1:4Xh3bDzO29j4TWNOI+24ubc0vbVFMg2PMnXKxK54/CA= github.com/go-task/slim-sprig v2.20.0+incompatible/go.mod h1:N/mhXZITr/EQAOErEHciKvO1bFei2Lld2Ym6h96pdy0= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -116,6 +133,18 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/gonvenience/bunt v1.3.5 h1:wSQquifvwEWtzn27k1ngLfeLaStyt0k1b/K6TrlCNAs= +github.com/gonvenience/bunt v1.3.5/go.mod h1:7ApqkVBEWvX04oJ28Q2WeI/BvJM6VtukaJAU/q/pTs8= +github.com/gonvenience/neat v1.3.12 h1:xwIyRbJcG9LgcDYys+HHLH9DqqHeQsUpS5CfBUeskbs= +github.com/gonvenience/neat v1.3.12/go.mod h1:8OljAIgPelN0uPPO94VBqxK+Kz98d6ZFwHDg5o/PfkE= +github.com/gonvenience/term v1.0.2 h1:qKa2RydbWIrabGjR/fegJwpW5m+JvUwFL8mLhHzDXn0= +github.com/gonvenience/term v1.0.2/go.mod h1:wThTR+3MzWtWn7XGVW6qQ65uaVf8GHED98KmwpuEQeo= +github.com/gonvenience/text v1.0.7 h1:YmIqmgTwxnACYCG59DykgMbomwteYyNhAmEUEJtPl14= +github.com/gonvenience/text v1.0.7/go.mod h1:OAjH+mohRszffLY6OjgQcUXiSkbrIavooFpfIt1ZwAs= +github.com/gonvenience/wrap v1.1.2 h1:xPKxNwL1HCguwyM+HlP/1CIuc9LRd7k8RodLwe9YTZA= +github.com/gonvenience/wrap v1.1.2/go.mod h1:GiryBSXoI3BAAhbWD1cZVj7RZmtiu0ERi/6R6eJfslI= +github.com/gonvenience/ytbx v1.4.4 h1:jQopwyaLsVGuwdxSiN4WkXjsEaFNPJ3V4lUj7eyEpzo= +github.com/gonvenience/ytbx v1.4.4/go.mod h1:w37+MKCPcCMY/jpPNmEklD4xKqrOAVBO6kIWW2+uI6M= github.com/google/certificate-transparency-go v1.1.7 h1:IASD+NtgSTJLPdzkthwvAG1ZVbF2WtFg4IvoA68XGSw= github.com/google/certificate-transparency-go v1.1.7/go.mod h1:FSSBo8fyMVgqptbfF6j5p/XNdgQftAhSmXcIxV9iphE= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -140,8 +169,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= @@ -154,18 +183,35 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro= +github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= +github.com/gruntwork-io/terratest v0.46.13 h1:FDaEoZ7DtkomV8pcwLdBV/VsytdjnPRqJkIriYEYwjs= +github.com/gruntwork-io/terratest v0.46.13/go.mod h1:8sxu3Qup8TxtbzOHzq0MUrQffJj/G61/OwlsReaCwpo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/homeport/dyff v1.6.0 h1:AN+ikld0Fy+qx34YE7655b/bpWuxS6cL9k852pE2GUc= +github.com/homeport/dyff v1.6.0/go.mod h1:FlAOFYzeKvxmU5nTrnG+qrlJVWpsFew7pt8L99p5q8k= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -194,10 +240,28 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 h1:BXxTozrOU8zgC5dkpn3J6NTRdoP+hjok/e+ACr4Hibk= +github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3/go.mod h1:x1uk6vxTiVuNt6S5R2UYgdhpj3oKojXvOXauHZ7dEnI= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= +github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -222,7 +286,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= @@ -241,6 +308,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= +github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -255,12 +324,16 @@ github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2z github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -281,8 +354,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U= +github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8= github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -290,6 +366,11 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2ZvtT+vQLY= @@ -396,7 +477,9 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -477,6 +560,7 @@ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGm google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -484,6 +568,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/modules/cli/cli_test.go b/modules/cli/cli_test.go index b7ea907be..c2a9b3714 100644 --- a/modules/cli/cli_test.go +++ b/modules/cli/cli_test.go @@ -66,7 +66,7 @@ func TestParse(t *testing.T) { "--metrics-bind-address=:18080", }, envVars: map[string]string{ - "GATEWAY_OPERATOR_METRIC_BIND_ADDRESS": ":28080", + "GATEWAY_OPERATOR_METRICS_BIND_ADDRESS": ":28080", "GATEWAY_OPERATOR_HEALTH_PROBE_BIND_ADDRESS": ":28081", }, expectedCfg: func() manager.Config { diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 3ca03d92a..63e99422c 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -91,7 +91,8 @@ type TestEnvironment struct { type TestEnvOption func(opt *testEnvOptions) type testEnvOptions struct { - Image string + Image string + InstallViaKustomize bool } // WithOperatorImage allows configuring the operator image to use in the test environment. @@ -101,9 +102,18 @@ func WithOperatorImage(image string) TestEnvOption { } } +// WithInstallViaKustomize makes the test environment install the operator and all the +// dependencies via kustomize. +func WithInstallViaKustomize() TestEnvOption { + return func(opts *testEnvOptions) { + opts.InstallViaKustomize = true + } +} + var loggerOnce sync.Once // CreateEnvironment creates a new independent testing environment for running isolated e2e test. +// When running with helm caller is responsible for cleaning up the environment. func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment { t.Helper() var opt testEnvOptions @@ -154,12 +164,21 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) if len(opt.Image) == 0 { opt.Image = getOperatorImage(t) } - kustomizeDir := PrepareKustomizeDir(t, opt.Image) + + var kustomizeDir KustomizeDir + if opt.InstallViaKustomize { + kustomizeDir = PrepareKustomizeDir(t, opt.Image) + } env, err := builder.Build(ctx) require.NoError(t, err) + t.Cleanup(func() { - cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests()) + if opt.InstallViaKustomize { + cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests()) + } else { //nolint:revive,staticcheck + // TODO: using helm for installation, don't clean up because we do not know the release name. + } }) t.Logf("waiting for cluster %s and all addons to become ready", env.Cluster().Name()) @@ -193,28 +212,32 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) require.NoError(t, operatorv1alpha1.AddToScheme(clients.MgrClient.Scheme())) require.NoError(t, operatorv1beta1.AddToScheme(clients.MgrClient.Scheme())) - t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL) - require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL)) + if opt.InstallViaKustomize { + t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL) + require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL)) - kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion) - t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL) - require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL)) + kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion) + t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL) + require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL)) - t.Log("creating system namespaces and serviceaccounts") - require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system")) + t.Log("creating system namespaces and serviceaccounts") + require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system")) - t.Log("deploying operator CRDs to test cluster via kustomize") - require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side")) + t.Log("deploying operator CRDs to test cluster via kustomize") + require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side")) - t.Log("deploying operator to test cluster via kustomize") - require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side")) + t.Log("deploying operator to test cluster via kustomize") + require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side")) - t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, clients.K8sClient)) + t.Log("waiting for operator deployment to complete") + require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient)) - t.Log("waiting for operator webhook service to be connective") - require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient), - webhookReadinessTimeout, webhookReadinessTick) + t.Log("waiting for operator webhook service to be connective") + require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient), + webhookReadinessTimeout, webhookReadinessTick) + } else { + t.Log("not deploying operator to test cluster via kustomize") + } t.Log("environment is ready, starting tests") @@ -269,17 +292,26 @@ func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentA } } -func waitForOperatorDeployment(ctx context.Context, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error { +func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error { outer: for { select { case <-ctx.Done(): return ctx.Err() default: - deployment, err := k8sClient.AppsV1().Deployments("kong-system").Get(ctx, "gateway-operator-controller-manager", metav1.GetOptions{}) + listOpts := metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=gateway-operator", + } + deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts) if err != nil { return err } + if len(deploymentList.Items) == 0 { + continue + } + + deployment := &deploymentList.Items[0] + if deployment.Status.AvailableReplicas <= 0 { continue } diff --git a/test/e2e/test_helm_install_upgrade.go b/test/e2e/test_helm_install_upgrade.go new file mode 100644 index 000000000..8fa13b85e --- /dev/null +++ b/test/e2e/test_helm_install_upgrade.go @@ -0,0 +1,146 @@ +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/gruntwork-io/terratest/modules/helm" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + + "github.com/kong/gateway-operator/pkg/utils/test" +) + +func init() { + addTestsToTestSuite(TestUpgrade) +} + +func TestUpgrade(t *testing.T) { + const ( + // Rel: https://github.com/Kong/charts/tree/main/charts/gateway-operator + chart = "kong/gateway-operator" + image = "docker.io/kong/gateway-operator-oss" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // createEnvironment will queue up environment cleanup if necessary + // and dumping diagnostics if the test fails. + e := CreateEnvironment(t, ctx) + + testcases := []struct { + name string + fromVersion string + toVersion string + upgradeToCurrent bool + assertions func(*testing.T, *test.K8sClients) + }{ + // NOTE: We do not support versions earlier than 1.2 with the helm chart. + // The initial version of the chart contained CRDs from KGO 1.2. which + // introduced a breaking change which makes it impossible to upgrade from + // automatically (without manually deleting the CRDs). + { + name: "upgrade from 1.2.0 to 1.2.3", + fromVersion: "1.2.1", + toVersion: "1.2.3", + assertions: func(t *testing.T, c *test.K8sClients) { + // TODO + }, + }, + { + name: "upgrade from 1.2.3 to current", + fromVersion: "1.2.3", + upgradeToCurrent: true, + }, + } + + var currentTag string + if imageLoad != "" { + t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q", imageLoad) + currentTag = vFromImage(t, imageLoad) + } else if imageOverride != "" { + t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q", imageOverride) + currentTag = vFromImage(t, imageOverride) + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var tag string + if tc.upgradeToCurrent { + if currentTag == "" { + t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" + + " env specified. Please specify the image to upgrade to in order to run this test.") + } + tag = currentTag + } else { + tag = tc.toVersion + } + + tagInReleaseName := tag + if len(tag) > 8 { + tagInReleaseName = tag[:8] + } + releaseName := strings.ReplaceAll(fmt.Sprintf("kgo-%s-to-%s", tc.fromVersion, tagInReleaseName), ".", "-") + values := map[string]string{ + "image.tag": tc.fromVersion, + "image.repository": image, + } + + opts := &helm.Options{ + KubectlOptions: &k8s.KubectlOptions{ + Namespace: e.Namespace.Name, + RestConfig: e.Environment.Cluster().Config(), + }, + SetValues: values, + } + + require.NoError(t, helm.AddRepoE(t, opts, "kong", "https://charts.konghq.com")) + require.NoError(t, helm.InstallE(t, opts, chart, releaseName)) + t.Cleanup(func() { + out, err := helm.RunHelmCommandAndGetOutputE(t, opts, "uninstall", releaseName) + if !assert.NoError(t, err) { + t.Logf("output: %s", out) + } + }) + + require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, + deploymentAssertConditions(deploymentReadyConditions()...), + )) + + opts.SetValues["image.tag"] = tag + + require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName)) + require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, + deploymentAssertConditions(deploymentReadyConditions()...), + )) + }) + } +} + +func deploymentReadyConditions() []appsv1.DeploymentCondition { + return []appsv1.DeploymentCondition{ + { + Reason: "NewReplicaSetAvailable", + Status: "True", + Type: "Progressing", + }, + { + Reason: "MinimumReplicasAvailable", + Status: "True", + Type: "Available", + }, + } +} + +func vFromImage(t *testing.T, image string) string { + splitImage := strings.Split(image, ":") + if len(splitImage) != 2 { + t.Fatalf("image %q does not contain a tag", image) + } + return splitImage[1] +} diff --git a/test/e2e/test_operator_logs.go b/test/e2e/test_operator_logs.go index 23f30f2e9..893410deb 100644 --- a/test/e2e/test_operator_logs.go +++ b/test/e2e/test_operator_logs.go @@ -72,7 +72,7 @@ func TestOperatorLogs(t *testing.T) { // createEnvironment will queue up environment cleanup if necessary // and dumping diagnostics if the test fails. - e := CreateEnvironment(t, ctx) + e := CreateEnvironment(t, ctx, WithInstallViaKustomize()) clients, testNamespace, cleaner := e.Clients, e.Namespace, e.Cleaner t.Log("finding the Pod for the Gateway Operator") diff --git a/test/e2e/test_upgrade.go b/test/e2e/test_upgrade.go index c403485cd..b6129871d 100644 --- a/test/e2e/test_upgrade.go +++ b/test/e2e/test_upgrade.go @@ -76,13 +76,13 @@ func testManifestsUpgrade( ctx context.Context, testParams upgradeTestParams, ) { - e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage)) + e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize()) kustomizationDir := PrepareKustomizeDir(t, testParams.toImage) t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name()) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, e.Clients.K8sClient, + require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient, deploymentAssertConditions( appsv1.DeploymentCondition{ Reason: "NewReplicaSetAvailable", diff --git a/test/e2e/test_webhook.go b/test/e2e/test_webhook.go index 49ff4782c..242a095a2 100644 --- a/test/e2e/test_webhook.go +++ b/test/e2e/test_webhook.go @@ -23,7 +23,7 @@ func TestDataPlaneValidatingWebhook(t *testing.T) { // createEnvironment will queue up environment cleanup if necessary // and dumping diagnostics if the test fails. - e := CreateEnvironment(t, ctx) + e := CreateEnvironment(t, ctx, WithInstallViaKustomize()) clients, testNamespace := e.Clients, e.Namespace testCases := []struct { diff --git a/test/helpers/setup.go b/test/helpers/setup.go index d16a3b410..9c473a6e0 100644 --- a/test/helpers/setup.go +++ b/test/helpers/setup.go @@ -75,14 +75,20 @@ func ParseGoTestFlags(testRunner func(t *testing.T), testSuiteToRun []func(t *te os.Exit(1) } + fmt.Printf("test.run: %s\n", fRun.Value.String()) + // Hack - set test.run flag to the name of the test function that runs the test suite // to execute it with tests that are returned from this function. // They are explicitly passed to RunTestSuite(...) as an argument. - if err := fRun.Value.Set(getFunctionName(testRunner)); err != nil { + if err := fRun.Value.Set(getFunctionName(testRunner) + "/" + fRun.Value.String()); err != nil { fmt.Println("testing: unexpected error happened (it should never happen, check the code)", err) os.Exit(1) } + fmt.Printf("test.run: %s\n", fRun.Value.String()) + + // os.Exit(0) + return testSuiteToRun } From 89b1a7fa3e861a3c4dcdd0bb658eee0f33b6e836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 29 Apr 2024 13:39:13 +0200 Subject: [PATCH 2/8] chore(tests): add logs in e2e tests --- config/manager/manager.yaml | 1 + test/e2e/environment.go | 87 +++++++- test/e2e/test_helm_install_upgrade.go | 301 ++++++++++++++++++++++++-- test/e2e/test_upgrade.go | 9 +- 4 files changed, 366 insertions(+), 32 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 43ccfa9e0..6421eb12b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,6 +12,7 @@ metadata: namespace: system labels: control-plane: controller-manager + app.kubernetes.io/name: gateway-operator spec: strategy: type: Recreate diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 63e99422c..d25536192 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -116,6 +116,11 @@ var loggerOnce sync.Once // When running with helm caller is responsible for cleaning up the environment. func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment { t.Helper() + + const ( + waitTime = 1 * time.Minute + ) + var opt testEnvOptions for _, o := range opts { o(&opt) @@ -230,7 +235,7 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient)) + require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", clients.K8sClient, waitTime)) t.Log("waiting for operator webhook service to be connective") require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient), @@ -280,26 +285,55 @@ func cleanupEnvironment(t *testing.T, ctx context.Context, env environments.Envi type deploymentAssertOptions func(*appsv1.Deployment) bool -func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentAssertOptions { +func deploymentAssertConditions(t *testing.T, conds ...appsv1.DeploymentCondition) deploymentAssertOptions { + t.Helper() + return func(deployment *appsv1.Deployment) bool { return lo.EveryBy(conds, func(cond appsv1.DeploymentCondition) bool { - return lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool { + if !lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool { return c.Type == cond.Type && c.Status == cond.Status && c.Reason == cond.Reason - }) + }) { + t.Logf("Deployment %s/%s does not have condition %#v", deployment.Namespace, deployment.Name, cond) + t.Logf("Deployment %s/%s current status: %#v", deployment.Namespace, deployment.Name, deployment.Status) + return false + } + return true }) } } -func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error { -outer: +func waitForOperatorDeployment( + t *testing.T, + ctx context.Context, + ns string, + k8sClient *kubernetes.Clientset, + waitTime time.Duration, + opts ...deploymentAssertOptions, +) error { + t.Helper() + + timer := time.NewTimer(waitTime) + defer timer.Stop() + pollTimer := time.NewTicker(time.Second) + defer pollTimer.Stop() + for { select { + case <-timer.C: + logOperatorPodLogs(t, ctx, k8sClient, ns) + return fmt.Errorf("timed out waiting for operator deployment in namespace %s", ns) case <-ctx.Done(): + logOperatorPodLogs(t, context.Background(), k8sClient, ns) return ctx.Err() - default: + case <-pollTimer.C: listOpts := metav1.ListOptions{ + // NOTE: This is a common label used by: + // - kustomize https://github.com/Kong/gateway-operator/blob/f98ef9358078ac100e143ab677a9ca836d0222a0/config/manager/manager.yaml#L15 + // - helm https://github.com/Kong/charts/blob/4968b34ae7c252ab056b37cc137eaeb7a071e101/charts/gateway-operator/templates/deployment.yaml#L5-L6 + // + // As long as kustomize is used for tests let's use this label selector. LabelSelector: "app.kubernetes.io/name=gateway-operator", } deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts) @@ -307,18 +341,20 @@ outer: return err } if len(deploymentList.Items) == 0 { + t.Logf("No operator deployment found in namespace %s", ns) continue } deployment := &deploymentList.Items[0] if deployment.Status.AvailableReplicas <= 0 { + t.Logf("Deployment %s/%s has no AvailableReplicas", ns, deployment.Name) continue } for _, opt := range opts { if !opt(deployment) { - continue outer + continue } } return nil @@ -326,6 +362,41 @@ outer: } } +func logOperatorPodLogs(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset, ns string) { + t.Helper() + + pods, err := k8sClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=gateway-operator", + }) + if err != nil { + t.Logf("Failed to list operator pods in namespace %s: %v", ns, err) + return + } + + if len(pods.Items) == 0 { + t.Logf("No operator pod found in namespace %s", ns) + return + } + + result := k8sClient.CoreV1().Pods(ns).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{ + Container: "manager", + InsecureSkipTLSVerifyBackend: true, + }).Do(ctx) + + if result.Error() != nil { + t.Logf("Failed to get logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, result.Error()) + return + } + + b, err := result.Raw() + if err != nil { + t.Logf("Failed to read logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, err) + return + } + + t.Logf("Operator pod logs:\n%s", string(b)) +} + func waitForOperatorWebhookEventually(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset) func() bool { return func() bool { if err := waitForOperatorWebhook(ctx, k8sClient); err != nil { diff --git a/test/e2e/test_helm_install_upgrade.go b/test/e2e/test_helm_install_upgrade.go index 8fa13b85e..a25f15a8b 100644 --- a/test/e2e/test_helm_install_upgrade.go +++ b/test/e2e/test_helm_install_upgrade.go @@ -5,25 +5,37 @@ import ( "fmt" "strings" "testing" + "time" "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/kong/gateway-operator/pkg/utils/test" + operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + "github.com/kong/gateway-operator/pkg/consts" + "github.com/kong/gateway-operator/pkg/utils/gateway" + testutils "github.com/kong/gateway-operator/pkg/utils/test" + "github.com/kong/gateway-operator/pkg/vars" + "github.com/kong/gateway-operator/test/helpers" ) func init() { - addTestsToTestSuite(TestUpgrade) + addTestsToTestSuite(TestHelmUpgrade) } -func TestUpgrade(t *testing.T) { +func TestHelmUpgrade(t *testing.T) { const ( // Rel: https://github.com/Kong/charts/tree/main/charts/gateway-operator chart = "kong/gateway-operator" image = "docker.io/kong/gateway-operator-oss" + + waitTime = 3 * time.Minute ) ctx, cancel := context.WithCancel(context.Background()) @@ -33,12 +45,19 @@ func TestUpgrade(t *testing.T) { // and dumping diagnostics if the test fails. e := CreateEnvironment(t, ctx) + // assertion is run after the upgrade to assert the state of the resources in the cluster. + type assertion struct { + Name string + Func func(*assert.CollectT, *testutils.K8sClients) + } + testcases := []struct { name string fromVersion string toVersion string + objectsToDeploy []client.Object upgradeToCurrent bool - assertions func(*testing.T, *test.K8sClients) + assertions []assertion }{ // NOTE: We do not support versions earlier than 1.2 with the helm chart. // The initial version of the chart contained CRDs from KGO 1.2. which @@ -46,37 +65,208 @@ func TestUpgrade(t *testing.T) { // automatically (without manually deleting the CRDs). { name: "upgrade from 1.2.0 to 1.2.3", - fromVersion: "1.2.1", + fromVersion: "1.2.0", toVersion: "1.2.3", - assertions: func(t *testing.T, c *test.K8sClients) { - // TODO + objectsToDeploy: []client.Object{ + &operatorv1beta1.GatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwconf-upgrade-120-123", + }, + Spec: baseGatewayConfigurationSpec(), + }, + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwclass-upgrade-120-123", + }, + Spec: gatewayv1.GatewayClassSpec{ + ParametersRef: &gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&e.Namespace.Name), + Name: "gwconf-upgrade-120-123", + }, + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + }, + }, + &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gw-upgrade-120-123-", + Labels: map[string]string{ + "gw-upgrade-120-123": "true", + }, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName("gwclass-upgrade-120-123"), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + }, + }, + assertions: []assertion{ + { + Name: "gateway is programmed", + Func: func(c *assert.CollectT, cl *testutils.K8sClients) { + gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: "gw-upgrade-120-123=true", + }) + require.NoError(c, err) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + }, + }, }, }, { name: "upgrade from 1.2.3 to current", fromVersion: "1.2.3", upgradeToCurrent: true, + objectsToDeploy: []client.Object{ + &operatorv1beta1.GatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwconf-upgrade-123-current", + }, + Spec: baseGatewayConfigurationSpec(), + }, + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwclass-upgrade-123-current", + }, + Spec: gatewayv1.GatewayClassSpec{ + ParametersRef: &gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&e.Namespace.Name), + Name: "gwconf-upgrade-123-current", + }, + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + }, + }, + &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gw-upgrade-123-current-", + Labels: map[string]string{ + "gw-upgrade-123-current": "true", + }, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName("gwclass-upgrade-123-current"), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + }, + }, + assertions: []assertion{ + { + Name: "gateway is programmed", + Func: func(c *assert.CollectT, cl *testutils.K8sClients) { + gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: "gw-upgrade-123-current=true", + }) + require.NoError(c, err) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + }, + }, + }, + }, + { + name: "upgrade from nightly to current", + fromVersion: "nightly", + upgradeToCurrent: true, + objectsToDeploy: []client.Object{ + &operatorv1beta1.GatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwconf-upgrade-nightly-current", + }, + Spec: baseGatewayConfigurationSpec(), + }, + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwclass-upgrade-nightly-to-current", + }, + Spec: gatewayv1.GatewayClassSpec{ + ParametersRef: &gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&e.Namespace.Name), + Name: "gwconf-upgrade-nightly-current", + }, + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + }, + }, + &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gw-upgrade-nightly-to-current-", + Labels: map[string]string{ + "gw-upgrade-nightly-to-current": "true", + }, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName("gwclass-upgrade-nightly-to-current"), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + }, + }, + assertions: []assertion{ + { + Name: "gateway is programmed", + Func: func(c *assert.CollectT, cl *testutils.K8sClients) { + gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: "gw-upgrade-nightly-to-current=true", + }) + require.NoError(c, err) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + }, + }, + }, }, } - var currentTag string + var ( + currentRepository string + currentTag string + ) if imageLoad != "" { t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q", imageLoad) - currentTag = vFromImage(t, imageLoad) + currentRepository, currentTag = splitRepoVersionFromImage(t, imageLoad) } else if imageOverride != "" { t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q", imageOverride) - currentTag = vFromImage(t, imageOverride) + currentRepository, currentTag = splitRepoVersionFromImage(t, imageOverride) } for _, tc := range testcases { + tc := tc t.Run(tc.name, func(t *testing.T) { - var tag string + var ( + tag string + targetRepository = image + ) if tc.upgradeToCurrent { if currentTag == "" { - t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" + - " env specified. Please specify the image to upgrade to in order to run this test.") + t.Skip( + "No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD env specified. " + + "Please specify the image to upgrade to in order to run this test.", + ) } tag = currentTag + targetRepository = currentRepository } else { tag = tc.toVersion } @@ -87,8 +277,13 @@ func TestUpgrade(t *testing.T) { } releaseName := strings.ReplaceAll(fmt.Sprintf("kgo-%s-to-%s", tc.fromVersion, tagInReleaseName), ".", "-") values := map[string]string{ - "image.tag": tc.fromVersion, - "image.repository": image, + "image.tag": tc.fromVersion, + "image.repository": image, + "readinessProbe.initialDelaySeconds": "1", + "readinessProbe.periodSeconds": "1", + // Disable leader election and anonymous reports for tests. + "no_leader_election": "true", + "anonymous_reports": "false", } opts := &helm.Options{ @@ -108,16 +303,35 @@ func TestUpgrade(t *testing.T) { } }) - require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, - deploymentAssertConditions(deploymentReadyConditions()...), + // Deploy the objects that should be present before the upgrade. + cl := client.NewNamespacedClient(e.Clients.MgrClient, e.Namespace.Name) + for _, obj := range tc.objectsToDeploy { + require.NoError(t, cl.Create(ctx, obj)) + t.Cleanup(func() { + require.NoError(t, client.IgnoreNotFound(cl.Delete(ctx, obj))) + }) + } + + require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, deploymentReadyConditions()...), )) opts.SetValues["image.tag"] = tag + opts.SetValues["image.repository"] = targetRepository require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName)) - require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, - deploymentAssertConditions(deploymentReadyConditions()...), - )) + require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, deploymentReadyConditions()...), + ), + ) + + for _, assertion := range tc.assertions { + t.Run(assertion.Name, func(t *testing.T) { + require.EventuallyWithT(t, func(c *assert.CollectT) { + assertion.Func(c, e.Clients) + }, waitTime, time.Second) + }) + } }) } } @@ -137,10 +351,53 @@ func deploymentReadyConditions() []appsv1.DeploymentCondition { } } -func vFromImage(t *testing.T, image string) string { +func splitRepoVersionFromImage(t *testing.T, image string) (string, string) { splitImage := strings.Split(image, ":") if len(splitImage) != 2 { t.Fatalf("image %q does not contain a tag", image) } - return splitImage[1] + return splitImage[0], splitImage[1] +} + +func baseGatewayConfigurationSpec() operatorv1beta1.GatewayConfigurationSpec { + return operatorv1beta1.GatewayConfigurationSpec{ + DataPlaneOptions: &operatorv1beta1.GatewayConfigDataPlaneOptions{ + Deployment: operatorv1beta1.DataPlaneDeploymentOptions{ + DeploymentOptions: operatorv1beta1.DeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: consts.DataPlaneProxyContainerName, + Image: helpers.GetDefaultDataPlaneImage(), + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: 1, + PeriodSeconds: 1, + }, + }, + }, + }, + }, + }, + }, + }, + ControlPlaneOptions: &operatorv1beta1.ControlPlaneOptions{ + Deployment: operatorv1beta1.ControlPlaneDeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: consts.ControlPlaneControllerContainerName, + Image: consts.DefaultControlPlaneImage, + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: 1, + PeriodSeconds: 1, + }, + }, + }, + }, + }, + }, + }, + } } diff --git a/test/e2e/test_upgrade.go b/test/e2e/test_upgrade.go index b6129871d..f945eeb59 100644 --- a/test/e2e/test_upgrade.go +++ b/test/e2e/test_upgrade.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/kong/kubernetes-testing-framework/pkg/clusters" "github.com/stretchr/testify/require" @@ -76,14 +77,18 @@ func testManifestsUpgrade( ctx context.Context, testParams upgradeTestParams, ) { + const ( + waitTime = 3 * time.Minute + ) + e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize()) kustomizationDir := PrepareKustomizeDir(t, testParams.toImage) t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name()) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient, - deploymentAssertConditions( + require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, appsv1.DeploymentCondition{ Reason: "NewReplicaSetAvailable", Status: "True", From 5891a1cd4c98a48110d2a8d5df4508659fd31a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 6 May 2024 18:10:21 +0200 Subject: [PATCH 3/8] Apply suggestions from code review Co-authored-by: Jakub Warczarek --- test/e2e/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/environment.go b/test/e2e/environment.go index d25536192..1ca83a263 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -113,7 +113,7 @@ func WithInstallViaKustomize() TestEnvOption { var loggerOnce sync.Once // CreateEnvironment creates a new independent testing environment for running isolated e2e test. -// When running with helm caller is responsible for cleaning up the environment. +// When running with Helm, the caller is responsible for cleaning up the environment. func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment { t.Helper() From 494b33263254da2941c955f6524573d4758ec8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 6 May 2024 18:11:14 +0200 Subject: [PATCH 4/8] chore: remove leftover debug code --- test/helpers/setup.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/helpers/setup.go b/test/helpers/setup.go index 9c473a6e0..50bb870ff 100644 --- a/test/helpers/setup.go +++ b/test/helpers/setup.go @@ -75,8 +75,6 @@ func ParseGoTestFlags(testRunner func(t *testing.T), testSuiteToRun []func(t *te os.Exit(1) } - fmt.Printf("test.run: %s\n", fRun.Value.String()) - // Hack - set test.run flag to the name of the test function that runs the test suite // to execute it with tests that are returned from this function. // They are explicitly passed to RunTestSuite(...) as an argument. @@ -85,10 +83,6 @@ func ParseGoTestFlags(testRunner func(t *testing.T), testSuiteToRun []func(t *te os.Exit(1) } - fmt.Printf("test.run: %s\n", fRun.Value.String()) - - // os.Exit(0) - return testSuiteToRun } From eb97dc31716c06868c76f0c05f251add12561fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 6 May 2024 18:13:26 +0200 Subject: [PATCH 5/8] chore: comment for InstallViaKustomize --- test/e2e/environment.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 1ca83a263..9603ba25e 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -91,7 +91,11 @@ type TestEnvironment struct { type TestEnvOption func(opt *testEnvOptions) type testEnvOptions struct { - Image string + Image string + // InstallViaKustomize makes the test environment install the operator and all the + // dependencies via kustomize. + // NOTE: when this is false the caller is responsible for installing the operator + // in the test environment. InstallViaKustomize bool } From 621149c3027be790264573cff2ac5ef8222b0fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 6 May 2024 18:21:28 +0200 Subject: [PATCH 6/8] chore: change comments, remove TODO --- test/e2e/environment.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 9603ba25e..b627b12ef 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -94,8 +94,8 @@ type testEnvOptions struct { Image string // InstallViaKustomize makes the test environment install the operator and all the // dependencies via kustomize. - // NOTE: when this is false the caller is responsible for installing the operator - // in the test environment. + // NOTE: when this is false the caller is responsible for installing (and cleaning up) + // the operator in the test environment. InstallViaKustomize bool } @@ -185,8 +185,6 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) t.Cleanup(func() { if opt.InstallViaKustomize { cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests()) - } else { //nolint:revive,staticcheck - // TODO: using helm for installation, don't clean up because we do not know the release name. } }) From d153f0aefe183d03d7b15b8489fdef1dce0775bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 6 May 2024 18:42:42 +0200 Subject: [PATCH 7/8] chore: add comment about using renovate --- test/e2e/test_helm_install_upgrade.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/test_helm_install_upgrade.go b/test/e2e/test_helm_install_upgrade.go index a25f15a8b..5e100485c 100644 --- a/test/e2e/test_helm_install_upgrade.go +++ b/test/e2e/test_helm_install_upgrade.go @@ -122,6 +122,8 @@ func TestHelmUpgrade(t *testing.T) { }, }, { + // TODO: use renovate to bump the version in these 2 lines. + // https://github.com/Kong/gateway-operator/issues/121 name: "upgrade from 1.2.3 to current", fromVersion: "1.2.3", upgradeToCurrent: true, From 7d3abc1a31e7e1a839040017c233897ad8beac97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 6 May 2024 19:12:15 +0200 Subject: [PATCH 8/8] chore: extract the gateway predicate cration to a function --- test/e2e/test_helm_install_upgrade.go | 75 +++++++++++++++------------ 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/test/e2e/test_helm_install_upgrade.go b/test/e2e/test_helm_install_upgrade.go index 5e100485c..75ba0ffd6 100644 --- a/test/e2e/test_helm_install_upgrade.go +++ b/test/e2e/test_helm_install_upgrade.go @@ -14,6 +14,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -52,12 +53,12 @@ func TestHelmUpgrade(t *testing.T) { } testcases := []struct { - name string - fromVersion string - toVersion string - objectsToDeploy []client.Object - upgradeToCurrent bool - assertions []assertion + name string + fromVersion string + toVersion string + objectsToDeploy []client.Object + upgradeToCurrent bool + assertionsAfterUpgrade []assertion }{ // NOTE: We do not support versions earlier than 1.2 with the helm chart. // The initial version of the chart contained CRDs from KGO 1.2. which @@ -105,18 +106,11 @@ func TestHelmUpgrade(t *testing.T) { }, }, }, - assertions: []assertion{ + assertionsAfterUpgrade: []assertion{ { Name: "gateway is programmed", Func: func(c *assert.CollectT, cl *testutils.K8sClients) { - gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "gw-upgrade-120-123=true", - }) - require.NoError(c, err) - require.Len(c, gws.Items, 1) - gw := &gws.Items[0] - assert.True(c, gateway.IsProgrammed(gw)) - assert.True(c, gateway.AreListenersProgrammed(gw)) + gatewayAndItsListenersAreProgrammedAssertion("gw-upgrade-120-123=true")(ctx, c, cl.MgrClient) }, }, }, @@ -165,18 +159,11 @@ func TestHelmUpgrade(t *testing.T) { }, }, }, - assertions: []assertion{ + assertionsAfterUpgrade: []assertion{ { Name: "gateway is programmed", Func: func(c *assert.CollectT, cl *testutils.K8sClients) { - gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "gw-upgrade-123-current=true", - }) - require.NoError(c, err) - require.Len(c, gws.Items, 1) - gw := &gws.Items[0] - assert.True(c, gateway.IsProgrammed(gw)) - assert.True(c, gateway.AreListenersProgrammed(gw)) + gatewayAndItsListenersAreProgrammedAssertion("gw-upgrade-123-current=true")(ctx, c, cl.MgrClient) }, }, }, @@ -223,18 +210,11 @@ func TestHelmUpgrade(t *testing.T) { }, }, }, - assertions: []assertion{ + assertionsAfterUpgrade: []assertion{ { Name: "gateway is programmed", Func: func(c *assert.CollectT, cl *testutils.K8sClients) { - gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "gw-upgrade-nightly-to-current=true", - }) - require.NoError(c, err) - require.Len(c, gws.Items, 1) - gw := &gws.Items[0] - assert.True(c, gateway.IsProgrammed(gw)) - assert.True(c, gateway.AreListenersProgrammed(gw)) + gatewayAndItsListenersAreProgrammedAssertion("gw-upgrade-nightly-to-current=true")(ctx, c, cl.MgrClient) }, }, }, @@ -327,7 +307,7 @@ func TestHelmUpgrade(t *testing.T) { ), ) - for _, assertion := range tc.assertions { + for _, assertion := range tc.assertionsAfterUpgrade { t.Run(assertion.Name, func(t *testing.T) { require.EventuallyWithT(t, func(c *assert.CollectT) { assertion.Func(c, e.Clients) @@ -403,3 +383,30 @@ func baseGatewayConfigurationSpec() operatorv1beta1.GatewayConfigurationSpec { }, } } + +// gatewayAndItsListenersAreProgrammedAssertion returns a predicate that checks +// if the Gateway and its listeners are programmed. +func gatewayAndItsListenersAreProgrammedAssertion(labelSelector string) func(context.Context, *assert.CollectT, client.Client) { + return func(ctx context.Context, c *assert.CollectT, cl client.Client) { + var gws gatewayv1.GatewayList + lReq, err := labels.ParseToRequirements(labelSelector) + if err != nil { + c.Errorf("failed to parse label selector %q: %v", labelSelector, err) + c.FailNow() + } + lSel := labels.NewSelector() + for _, req := range lReq { + lSel = lSel.Add(req) + } + + require.NoError(c, + cl.List(ctx, &gws, &client.ListOptions{ + LabelSelector: lSel, + }), + ) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + } +}