From b86d182268555bfb4685fe0135fa788de25e695a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 11 Aug 2023 11:10:02 -0400 Subject: [PATCH 001/116] bump kafka-go to include acl apis --- go.mod | 10 +++++----- go.sum | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 4293590e..294f18c6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.35 + github.com/segmentio/kafka-go v0.4.42 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 @@ -25,7 +25,7 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.15.7 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -39,9 +39,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/xdg/scram v1.0.5 // indirect github.com/xdg/stringprep v1.0.3 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 4e4f327e..43f1bcdd 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -76,6 +78,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= github.com/segmentio/kafka-go v0.4.35 h1:TAsQ7q1SjS39PcFvU0zDJhCuVAxHomy7xOAfbdSuhzs= github.com/segmentio/kafka-go v0.4.35/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= +github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= +github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= @@ -93,6 +97,9 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= @@ -116,6 +123,7 @@ golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -137,16 +145,20 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From f1ec537ab4165dac41eaedac65ae97e4dae5cfab Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 11 Aug 2023 11:12:07 -0400 Subject: [PATCH 002/116] add acl interfaces and aclinfo type stub --- pkg/admin/client.go | 20 ++++++++++++++++++++ pkg/admin/types.go | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 733a9ab0..46f3fbdf 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -94,4 +94,24 @@ type Client interface { // Close closes the client. Close() error + + // GetACLs gets full information about each ACL in the cluster. + GetACLs( + ctx context.Context, + names []string, + detailed bool, + ) ([]ACLInfo, error) + + // GetACL gets the details of a single ACL in the cluster. + GetACL( + ctx context.Context, + name string, + detailed bool, + ) (ACLInfo, error) + + // CreateACL creates an ACL in the cluster. + CreateACL( + ctx context.Context, + config kafka.ACLEntry, + ) error } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 0221a39b..10a2c3d3 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -792,3 +792,7 @@ func NewLeaderPartitions( return newLeaderPartitions } + +// ACLInfo represents the information stored about an ACL in zookeeper. +type ACLInfo struct { +} From 07a63c7815873d1fb60e109893924b4370185c3b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 11 Aug 2023 11:52:07 -0400 Subject: [PATCH 003/116] pull latest kafka-go and use kafka-go aclresource type --- go.mod | 2 +- go.sum | 2 ++ pkg/admin/client.go | 4 ++-- pkg/admin/types.go | 4 ---- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 294f18c6..f3e7e4ff 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.42 + github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 43f1bcdd..7bbae149 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/segmentio/kafka-go v0.4.35 h1:TAsQ7q1SjS39PcFvU0zDJhCuVAxHomy7xOAfbdS github.com/segmentio/kafka-go v0.4.35/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= +github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= +github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 46f3fbdf..45c467c7 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -100,14 +100,14 @@ type Client interface { ctx context.Context, names []string, detailed bool, - ) ([]ACLInfo, error) + ) ([]kafka.ACLResource, error) // GetACL gets the details of a single ACL in the cluster. GetACL( ctx context.Context, name string, detailed bool, - ) (ACLInfo, error) + ) (kafka.ACLResource, error) // CreateACL creates an ACL in the cluster. CreateACL( diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 10a2c3d3..0221a39b 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -792,7 +792,3 @@ func NewLeaderPartitions( return newLeaderPartitions } - -// ACLInfo represents the information stored about an ACL in zookeeper. -type ACLInfo struct { -} From 474c260efbc47cb8b622ca3da223def3daab714d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 8 Sep 2023 09:25:34 -0400 Subject: [PATCH 004/116] wip --- docker-compose-auth.yml | 2 ++ go.mod | 5 +-- go.sum | 23 ++++---------- pkg/admin/brokerclient.go | 49 ++++++++++++++++++++++++++++ pkg/admin/brokerclient_test.go | 58 ++++++++++++++++++++++++++++++++++ pkg/admin/client.go | 20 ------------ pkg/util/testing.go | 10 ++++++ 7 files changed, 129 insertions(+), 38 deletions(-) diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index 16b70690..7897cb8d 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -34,6 +34,8 @@ services: KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095" KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512" + KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' KAFKA_SSL_KEYSTORE_LOCATION: /certs/kafka.keystore.jks KAFKA_SSL_KEYSTORE_PASSWORD: test123 KAFKA_SSL_KEY_PASSWORD: test123 diff --git a/go.mod b/go.mod index f3e7e4ff..7f969696 100644 --- a/go.mod +++ b/go.mod @@ -37,8 +37,9 @@ require ( github.com/pkg/term v0.0.0-20200520122047-c3ffed290a03 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/xdg/scram v1.0.5 // indirect - github.com/xdg/stringprep v1.0.3 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/go.sum b/go.sum index 7bbae149..4d6c3b80 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= -github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -76,10 +74,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= -github.com/segmentio/kafka-go v0.4.35 h1:TAsQ7q1SjS39PcFvU0zDJhCuVAxHomy7xOAfbdSuhzs= -github.com/segmentio/kafka-go v0.4.35/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= -github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= -github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= @@ -99,15 +93,14 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= -github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= -github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -120,11 +113,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -145,21 +136,21 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 4bc4063b..4246ea7c 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -690,3 +690,52 @@ func configEntriesToAPIConfigs( return apiConfigs } + +// TODO: what fields should we let people filter on / how best to support that? +// It could be really useful to be able to use this to answer questions like what services have access topics x,y,z or +// who has write access to topic b? +// Is GetACL (single ACL) even applicable? +// GetACLs gets full information about each ACL in the cluster. +func (c *BrokerAdminClient) GetACLs( + ctx context.Context, + filter kafka.ACLFilter, +) ([]kafka.ACLResource, error) { + req := kafka.DescribeACLsRequest{ + Filter: filter, + } + log.Debugf("DescribeACLs request: %+v", req) + + resp, err := c.client.DescribeACLs(ctx, &req) + log.Debugf("DescribeACLs response: %+v (%+v)", resp, err) + if err != nil { + return nil, err + } + return resp.Resources, nil +} + +// CreateACL creates an ACL in the cluster. +func (c *BrokerAdminClient) CreateACL( + ctx context.Context, + entry kafka.ACLEntry, +) error { + if c.config.ReadOnly { + return errors.New("Cannot create ACL in read-only mode") + } + + req := kafka.CreateACLsRequest{ + ACLs: []kafka.ACLEntry{ + entry, + }, + } + log.Debugf("CreateACLs request: %+v", req) + + resp, err := c.client.CreateACLs(ctx, &req) + log.Debugf("CreateACLs response: %+v (%+v)", resp, err) + if err != nil { + return err + } + if len(resp.Errors) > 0 { + fmt.Errorf("%+v", resp.Errors) + } + return nil +} diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index d998df84..0c494ca3 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -574,3 +574,61 @@ func TestBrokerClientCreateTopicError(t *testing.T) { ) require.Error(t, err) } + +func TestBrokerClientCreateGetACL(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") + } + + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, + ) + require.NoError(t, err) + + principal := util.RandomString("User:user-create-", 6) + topicName := util.RandomString("topic-create-", 6) + + err = client.CreateACL( + ctx, + kafka.ACLEntry{ + Principal: principal, + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + ResourceType: kafka.ResourceTypeTopic, + ResourcePatternType: kafka.PatternTypeLiteral, + ResourceName: topicName, + Host: "*", + }, + ) + require.NoError(t, err) + + filter := kafka.ACLFilter{ + ResourceNameFilter: topicName, + } + + aclsInfo, err := client.GetACLs(ctx, filter) + require.NoError(t, err) + expected := []kafka.ACLResource{ + { + ResourceType: kafka.ResourceTypeTopic, + ResourceName: topicName, + PatternType: kafka.PatternTypeLiteral, + ACLs: []kafka.ACLDescription{ + { + Principal: principal, + Host: "*", + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + }, + }, + }, + } + assert.Equal(t, expected, aclsInfo) + +} diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 45c467c7..733a9ab0 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -94,24 +94,4 @@ type Client interface { // Close closes the client. Close() error - - // GetACLs gets full information about each ACL in the cluster. - GetACLs( - ctx context.Context, - names []string, - detailed bool, - ) ([]kafka.ACLResource, error) - - // GetACL gets the details of a single ACL in the cluster. - GetACL( - ctx context.Context, - name string, - detailed bool, - ) (kafka.ACLResource, error) - - // CreateACL creates an ACL in the cluster. - CreateACL( - ctx context.Context, - config kafka.ACLEntry, - ) error } diff --git a/pkg/util/testing.go b/pkg/util/testing.go index 2734086b..e81e412c 100644 --- a/pkg/util/testing.go +++ b/pkg/util/testing.go @@ -48,6 +48,16 @@ func CanTestBrokerAdmin() bool { return false } +// CanTestBrokerAdminSecurity returns whether we can test the broker-only admin client security features. +func CanTestBrokerAdminSecurity() bool { + value, ok := os.LookupEnv("KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY") + if ok && value != "" { + return true + } + + return false +} + var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") // RandomString returns a random string with the argument length. From 6e0ec36e1298e0e0aa72a3ea6103c7c5ad0986f0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 8 Sep 2023 11:21:29 -0400 Subject: [PATCH 005/116] fix test --- pkg/admin/brokerclient_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 0c494ca3..918c422e 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -577,7 +577,7 @@ func TestBrokerClientCreateTopicError(t *testing.T) { func TestBrokerClientCreateGetACL(t *testing.T) { if !util.CanTestBrokerAdminSecurity() { - t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") } ctx := context.Background() @@ -586,6 +586,12 @@ func TestBrokerClientCreateGetACL(t *testing.T) { BrokerAdminClientConfig{ ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), + SASL: SASLConfig{ + Enabled: true, + Mechanism: SASLMechanismScramSHA512, + Username: "adminscram", + Password: "admin-secret-512", + }, }, }, ) @@ -609,7 +615,11 @@ func TestBrokerClientCreateGetACL(t *testing.T) { require.NoError(t, err) filter := kafka.ACLFilter{ - ResourceNameFilter: topicName, + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, } aclsInfo, err := client.GetACLs(ctx, filter) From 7b9454d4f0d930e4c3c53fe5646e7daa486e5f53 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 11 Sep 2023 15:08:52 -0400 Subject: [PATCH 006/116] fix typos --- cmd/topicctl/subcmd/get.go | 6 +++- pkg/admin/brokerclient.go | 2 +- pkg/admin/brokerclient_test.go | 5 ++-- pkg/admin/client.go | 6 ++++ pkg/admin/format.go | 54 ++++++++++++++++++++++++++++++++++ pkg/admin/types.go | 12 ++++++++ pkg/admin/zkclient.go | 7 +++++ pkg/cli/cli.go | 17 ++++++++++- 8 files changed, 104 insertions(+), 5 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 804c851d..5b028058 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -17,7 +17,7 @@ var getCmd = &cobra.Command{ Long: strings.Join( []string{ "Get instances of a particular type.", - "Supported types currently include: balance, brokers, config, groups, lags, members, partitions, offsets, and topics.", + "Supported types currently include: balance, brokers, config, groups, lags, members, partitions, offsets, topics, and acls.", "", "See the tool README for a detailed description of each one.", }, @@ -140,6 +140,10 @@ func getRun(cmd *cobra.Command, args []string) error { } return cliRunner.GetTopics(ctx, getConfig.full) + case "acls": + // TODO: add arg validation once we figure out filtering args + + return cliRunner.GetAcls(ctx, nil) default: return fmt.Errorf("Unrecognized resource type: %s", resource) } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 4246ea7c..79c36b35 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -699,7 +699,7 @@ func configEntriesToAPIConfigs( func (c *BrokerAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, -) ([]kafka.ACLResource, error) { +) ([]ACLInfo, error) { req := kafka.DescribeACLsRequest{ Filter: filter, } diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 918c422e..ed8e053d 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -589,8 +589,9 @@ func TestBrokerClientCreateGetACL(t *testing.T) { SASL: SASLConfig{ Enabled: true, Mechanism: SASLMechanismScramSHA512, - Username: "adminscram", - Password: "admin-secret-512", + // TODO: don't hardcode these in tests, pull from env vars + Username: "adminscram", + Password: "admin-secret-512", }, }, }, diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 733a9ab0..c3b4bf8c 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -38,6 +38,12 @@ type Client interface { detailed bool, ) (TopicInfo, error) + // GetACLs gets full information about each ACL in the cluster. + GetACLs( + ctx context.Context, + filter kafka.ACLFilter, + ) ([]ACLInfo, error) + // UpdateTopicConfig updates the configuration for the argument topic. It returns the config // keys that were updated. UpdateTopicConfig( diff --git a/pkg/admin/format.go b/pkg/admin/format.go index c6b9c64f..ec7e4645 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -745,6 +745,60 @@ func FormatBrokerMaxPartitions( return string(bytes.TrimRight(buf.Bytes(), "\n")) } +// FormatAcls creates a pretty table that lists the details of the +// argument acls. +func FormatAcls(acls []ACLInfo) string { + buf := &bytes.Buffer{} + + headers := []string{ + "Resource Type", + "Resource Name", + "Principal", + "Host", + "Operation", + "Permission Type", + } + + table := tablewriter.NewWriter(buf) + table.SetHeader(headers) + table.SetAutoWrapText(false) + table.SetColumnAlignment( + []int{ + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + }, + ) + table.SetBorders( + tablewriter.Border{ + Left: false, + Top: true, + Right: false, + Bottom: true, + }, + ) + + for _, acl := range acls { + row := []string{ + // TODO: convert ints to something human readable + fmt.Sprintf("%d", acl.ResourceType), + acl.ResourceName, + acl.Principal, + acl.Host, + fmt.Sprintf("%d", acl.Operation), + fmt.Sprintf("%d", acl.PermissionType), + } + + table.Append(row) + } + + table.Render() + return string(bytes.TrimRight(buf.Bytes(), "\n")) +} + func prettyConfig(config map[string]string) string { rows := []string{} diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 0221a39b..6ab33ba8 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -8,6 +8,7 @@ import ( "strconv" "time" + "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/util" ) @@ -73,6 +74,17 @@ type PartitionAssignment struct { Replicas []int `json:"replicas"` } +// PartitionInfo represents the information stored about an ACL +// in zookeeper. +type ACLInfo struct { + ResourceType kafka.ResourceType + ResourceName string + Principal string + Host string + Operation kafka.ACLOperationType + PermissionType kafka.ACLPermissionType +} + type zkClusterID struct { Version string `json:"version"` ID string `json:"id"` diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 575dee8e..07168774 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -422,6 +422,13 @@ func (c *ZKAdminClient) GetTopic( return topics[0], nil } +func (c *ZKAdminClient) GetACLs( + ctx context.Context, + filter kafka.ACLFilter, +) ([]ACLInfo, error) { + return nil, nil +} + // UpdateTopicConfig updates the config JSON for a topic and sets a change // notification so that the brokers are notified. If overwrite is true, then // it will overwrite existing config entries. diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 7b0542c7..c0caa45c 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -476,7 +476,6 @@ func (c *CLIRunner) GetTopics(ctx context.Context, full bool) error { c.stopSpinner() return err } - brokers, err := c.adminClient.GetBrokers(ctx, nil) c.stopSpinner() if err != nil { return err @@ -552,6 +551,22 @@ func (c *CLIRunner) Tail( return err } +// TODO add options for filtering +// GetAcls fetches the details of each acl in the cluster and prints out a summary. +func (c *CLIRunner) GetAcls(ctx context.Context) error { + c.startSpinner() + + acls, err := c.adminClient.GetAcls(ctx, nil) + c.stopSpinner() + if err != nil { + return err + } + + c.printer("Acls:\n%s", admin.FormatAcls(acls)) + + return nil +} + func (c *CLIRunner) startSpinner() { if c.spinnerObj != nil { c.spinnerObj.Start() From 49f7e1936c9b91859b939b3e54281ea7faf86d8a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 11 Sep 2023 15:22:10 -0400 Subject: [PATCH 007/116] get acls working --- pkg/admin/brokerclient.go | 22 +++++++++++++++++++++- pkg/admin/brokerclient_test.go | 19 +++++++------------ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 79c36b35..6f0cf20a 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -710,7 +710,27 @@ func (c *BrokerAdminClient) GetACLs( if err != nil { return nil, err } - return resp.Resources, nil + + if resp.Error != nil { + return nil, resp.Error + } + + aclinfos := []ACLInfo{} + + for _, resource := range resp.Resources { + for _, acl := range resource.ACLs { + aclinfos = append(aclinfos, ACLInfo{ + ResourceType: resource.ResourceType, + ResourceName: resource.ResourceName, + Principal: acl.Principal, + Host: acl.Host, + Operation: acl.Operation, + PermissionType: acl.PermissionType, + }) + } + } + + return aclinfos, nil } // CreateACL creates an ACL in the cluster. diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index ed8e053d..d4b896e4 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -625,19 +625,14 @@ func TestBrokerClientCreateGetACL(t *testing.T) { aclsInfo, err := client.GetACLs(ctx, filter) require.NoError(t, err) - expected := []kafka.ACLResource{ + expected := []ACLInfo{ { - ResourceType: kafka.ResourceTypeTopic, - ResourceName: topicName, - PatternType: kafka.PatternTypeLiteral, - ACLs: []kafka.ACLDescription{ - { - Principal: principal, - Host: "*", - Operation: kafka.ACLOperationTypeRead, - PermissionType: kafka.ACLPermissionTypeAllow, - }, - }, + ResourceType: kafka.ResourceTypeTopic, + ResourceName: topicName, + Principal: principal, + Host: "*", + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, }, } assert.Equal(t, expected, aclsInfo) From 8382e98624ecc4efbb9813fe0c752b55c39afa5a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 11:21:36 -0400 Subject: [PATCH 008/116] getacls working --- cmd/topicctl/subcmd/get.go | 4 ++-- pkg/admin/brokerclient.go | 7 +++++++ pkg/admin/brokerclient_test.go | 1 + pkg/admin/support.go | 3 +++ pkg/admin/types.go | 1 + pkg/admin/zkclient.go | 4 +++- pkg/cli/cli.go | 13 ++++++++++--- 7 files changed, 27 insertions(+), 6 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 5b028058..7acc36b1 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -59,6 +59,7 @@ func getPreRun(cmd *cobra.Command, args []string) error { return getConfig.shared.validate() } +// TODO: make each of these "gets" a separate subcommand func getRun(cmd *cobra.Command, args []string) error { ctx := context.Background() sess := session.Must(session.NewSession()) @@ -142,8 +143,7 @@ func getRun(cmd *cobra.Command, args []string) error { return cliRunner.GetTopics(ctx, getConfig.full) case "acls": // TODO: add arg validation once we figure out filtering args - - return cliRunner.GetAcls(ctx, nil) + return cliRunner.GetACLs(ctx) default: return fmt.Errorf("Unrecognized resource type: %s", resource) } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 6f0cf20a..fc34e000 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -98,6 +98,12 @@ func NewBrokerAdminClient( if _, ok := maxVersions["AlterClientQuotas"]; ok { supportedFeatures.DynamicBrokerConfigs = true } + + // If we have DescribeAcls, than we're running a version of Kafka > 2.0.1, + // that will have support for all ACLs APIs. + if _, ok := maxVersions["DescribeAcls"]; ok { + supportedFeatures.ACLs = true + } log.Debugf("Supported features: %+v", supportedFeatures) adminClient := &BrokerAdminClient{ @@ -722,6 +728,7 @@ func (c *BrokerAdminClient) GetACLs( aclinfos = append(aclinfos, ACLInfo{ ResourceType: resource.ResourceType, ResourceName: resource.ResourceName, + PatternType: resource.PatternType, Principal: acl.Principal, Host: acl.Host, Operation: acl.Operation, diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index d4b896e4..2d8c21b6 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -629,6 +629,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { { ResourceType: kafka.ResourceTypeTopic, ResourceName: topicName, + PatternType: kafka.PatternTypeLiteral, Principal: principal, Host: "*", Operation: kafka.ACLOperationTypeRead, diff --git a/pkg/admin/support.go b/pkg/admin/support.go index 61c91118..242032b4 100644 --- a/pkg/admin/support.go +++ b/pkg/admin/support.go @@ -16,4 +16,7 @@ type SupportedFeatures struct { // DynamicBrokerConfigs indicates whether the client can return dynamic broker configs // like leader.replication.throttled.rate. DynamicBrokerConfigs bool + + // ACLs indicates whether the client supports access control levels. + ACLs bool } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 6ab33ba8..125f2160 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -79,6 +79,7 @@ type PartitionAssignment struct { type ACLInfo struct { ResourceType kafka.ResourceType ResourceName string + PatternType kafka.PatternType Principal string Host string Operation kafka.ACLOperationType diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 07168774..fa032764 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -763,12 +763,14 @@ func (c *ZKAdminClient) LockHeld( // GetSupportedFeatures returns the features that are supported by this client. func (c *ZKAdminClient) GetSupportedFeatures() SupportedFeatures { - // The zk-based client supports everything. + // The zk-based client supports everything except for ACLs. + // Zookeeper can support ACLs, topicctl just hasn't added support for it yet. return SupportedFeatures{ Reads: true, Applies: true, Locks: true, DynamicBrokerConfigs: true, + ACLs: false, } } diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index c0caa45c..68223baa 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -14,6 +14,7 @@ import ( "github.com/briandowns/spinner" "github.com/fatih/color" + "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/check" @@ -476,6 +477,7 @@ func (c *CLIRunner) GetTopics(ctx context.Context, full bool) error { c.stopSpinner() return err } + brokers, err := c.adminClient.GetBrokers(ctx, nil) c.stopSpinner() if err != nil { return err @@ -553,16 +555,21 @@ func (c *CLIRunner) Tail( // TODO add options for filtering // GetAcls fetches the details of each acl in the cluster and prints out a summary. -func (c *CLIRunner) GetAcls(ctx context.Context) error { +func (c *CLIRunner) GetACLs(ctx context.Context) error { c.startSpinner() - acls, err := c.adminClient.GetAcls(ctx, nil) + acls, err := c.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeAny, + ResourcePatternTypeFilter: kafka.PatternTypeAny, + Operation: kafka.ACLOperationTypeAny, + PermissionType: kafka.ACLPermissionTypeAny, + }) c.stopSpinner() if err != nil { return err } - c.printer("Acls:\n%s", admin.FormatAcls(acls)) + c.printer("ACLs:\n%s", admin.FormatAcls(acls)) return nil } From 7b8ee4282d309441a9586db13dde05c5ec30eff5 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 11:50:30 -0400 Subject: [PATCH 009/116] upgrade cobra to latest --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4293590e..0f232842 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/segmentio/kafka-go v0.4.35 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -23,7 +23,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/mattn/go-colorable v0.1.9 // indirect diff --git a/go.sum b/go.sum index 4e4f327e..b8fb8ecf 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -80,8 +80,8 @@ github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d07 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 2a7d2decaf30d7fd75b98852b8d0a76e724bb205 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 14:32:36 -0400 Subject: [PATCH 010/116] finish separating get into separate subcommands --- cmd/topicctl/subcmd/get.go | 316 ++++++++++++++++++++++++---------- cmd/topicctl/subcmd/shared.go | 28 +-- 2 files changed, 239 insertions(+), 105 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 804c851d..67f8332b 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -2,7 +2,6 @@ package subcmd import ( "context" - "fmt" "strings" "github.com/aws/aws-sdk-go/aws/session" @@ -17,15 +16,10 @@ var getCmd = &cobra.Command{ Long: strings.Join( []string{ "Get instances of a particular type.", - "Supported types currently include: balance, brokers, config, groups, lags, members, partitions, offsets, and topics.", - "", - "See the tool README for a detailed description of each one.", }, "\n", ), - Args: cobra.MinimumNArgs(1), - PreRunE: getPreRun, - RunE: getRun, + PersistentPreRunE: getPreRun, } type getCmdConfig struct { @@ -38,20 +32,30 @@ type getCmdConfig struct { var getConfig getCmdConfig func init() { - getCmd.Flags().BoolVar( + getCmd.PersistentFlags().BoolVar( &getConfig.full, "full", false, "Show more full information for resources", ) - getCmd.Flags().BoolVar( + getCmd.PersistentFlags().BoolVar( &getConfig.sortValues, "sort-values", false, "Sort by value instead of name; only applies for lags at the moment", ) - addSharedFlags(getCmd, &getConfig.shared) + getCmd.AddCommand( + balanceCmd(), + brokersCmd(), + configCmd(), + groupsCmd(), + lagsCmd(), + membersCmd(), + partitionsCmd(), + offsetsCmd(), + topicsCmd(), + ) RootCmd.AddCommand(getCmd) } @@ -59,88 +63,218 @@ func getPreRun(cmd *cobra.Command, args []string) error { return getConfig.shared.validate() } -func getRun(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) +func balanceCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "balance [optional topic]", + Short: "Number of replicas per broker position for topic or cluster as a whole", + Long: strings.Join([]string{ + "Displays the number of replicas per broker position.", + "Accepts an optional argument of a topic, which will just scope this to that topic. If topic is omitted, the balance displayed will be for the entire cluster.", + }, + "\n", + ), + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + + var topicName string + if len(args) == 1 { + topicName = args[0] + } + return cliRunner.GetBrokerBalance(ctx, topicName) + }, + PreRunE: getPreRun, + } + return cmd +} + +func brokersCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "brokers", + Short: "Displays descriptions of each broker in the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) - if err != nil { - return err + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetBrokers(ctx, getConfig.full) + }, } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) - - resource := args[0] - - switch resource { - case "balance": - var topicName string - - if len(args) == 2 { - topicName = args[1] - } else if len(args) > 2 { - return fmt.Errorf("Can provide at most one positional argument with brokers") - } - - return cliRunner.GetBrokerBalance(ctx, topicName) - case "brokers": - if len(args) > 1 { - return fmt.Errorf("Can only provide one positional argument with brokers") - } - - return cliRunner.GetBrokers(ctx, getConfig.full) - case "config": - if len(args) != 2 { - return fmt.Errorf("Must provide broker ID or topic name as second positional argument") - } - - return cliRunner.GetConfig(ctx, args[1]) - case "groups": - if len(args) > 1 { - return fmt.Errorf("Can only provide one positional argument with groups") - } - - return cliRunner.GetGroups(ctx) - case "lags": - if len(args) != 3 { - return fmt.Errorf("Must provide topic and groupID as additional positional arguments") - } - - return cliRunner.GetMemberLags( - ctx, - args[1], - args[2], - getConfig.full, - getConfig.sortValues, - ) - case "members": - if len(args) != 2 { - return fmt.Errorf("Must provide group ID as second positional argument") - } - - return cliRunner.GetGroupMembers(ctx, args[1], getConfig.full) - case "partitions": - if len(args) != 2 { - return fmt.Errorf("Must provide topic as second positional argument") - } - topicName := args[1] - - return cliRunner.GetPartitions(ctx, topicName) - case "offsets": - if len(args) != 2 { - return fmt.Errorf("Must provide topic as second positional argument") - } - topicName := args[1] - - return cliRunner.GetOffsets(ctx, topicName) - case "topics": - if len(args) > 1 { - return fmt.Errorf("Can only provide one positional argument with args") - } - - return cliRunner.GetTopics(ctx, getConfig.full) - default: - return fmt.Errorf("Unrecognized resource type: %s", resource) + return cmd +} + +func configCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config [broker or topic]", + Short: "Displays configuration for the provider broker or topic.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetConfig(ctx, args[0]) + }, + } + return cmd +} + +func groupsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "groups", + Short: "Displays consumer group informatin for the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetGroups(ctx) + }, + } + return cmd +} + +func lagsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "lags [topic] [group]", + Short: "Displays consumer group lag for the specified topic and consumer group.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetMemberLags( + ctx, + args[0], + args[1], + getConfig.full, + getConfig.sortValues, + ) + }, + } + return cmd +} + +func membersCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "members [group]", + Short: "Details of each member in the specified consumer group.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetGroupMembers(ctx, args[0], getConfig.full) + }, + } + return cmd +} + +func partitionsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "partitions [topic]", + Short: "Displays partition information for the specified topic.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetPartitions(ctx, args[0]) + }, + } + return cmd +} + +func offsetsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "offsets [topic]", + Short: "Displays offset information for the specified topic along with start and end times.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetOffsets(ctx, args[0]) + }, + } + return cmd +} + +func topicsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "topics", + Short: "Displays information for all topics in the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetTopics(ctx, getConfig.full) + }, } + return cmd } diff --git a/cmd/topicctl/subcmd/shared.go b/cmd/topicctl/subcmd/shared.go index a73f1e44..7f5d925c 100644 --- a/cmd/topicctl/subcmd/shared.go +++ b/cmd/topicctl/subcmd/shared.go @@ -173,88 +173,88 @@ func (s sharedOptions) getAdminClient( } func addSharedFlags(cmd *cobra.Command, options *sharedOptions) { - cmd.Flags().StringVarP( + cmd.PersistentFlags().StringVarP( &options.brokerAddr, "broker-addr", "b", "", "Broker address", ) - cmd.Flags().BoolVarP( + cmd.PersistentFlags().BoolVarP( &options.expandEnv, "expand-env", "", false, "Expand environment in cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.clusterConfig, "cluster-config", os.Getenv("TOPICCTL_CLUSTER_CONFIG"), "Cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.saslMechanism, "sasl-mechanism", "", "SASL mechanism if using SASL (choices: AWS-MSK-IAM, PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512)", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.saslPassword, "sasl-password", os.Getenv("TOPICCTL_SASL_PASSWORD"), "SASL password if using SASL; will override value set in cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.saslUsername, "sasl-username", os.Getenv("TOPICCTL_SASL_USERNAME"), "SASL username if using SASL; will override value set in cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsCACert, "tls-ca-cert", "", "Path to client CA cert PEM file if using TLS", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsCert, "tls-cert", "", "Path to client cert PEM file if using TLS", ) - cmd.Flags().BoolVar( + cmd.PersistentFlags().BoolVar( &options.tlsEnabled, "tls-enabled", false, "Use TLS for communication with brokers", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsKey, "tls-key", "", "Path to client private key PEM file if using TLS", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsServerName, "tls-server-name", "", "Server name to use for TLS cert verification", ) - cmd.Flags().BoolVar( + cmd.PersistentFlags().BoolVar( &options.tlsSkipVerify, "tls-skip-verify", false, "Skip hostname verification when using TLS", ) - cmd.Flags().StringVarP( + cmd.PersistentFlags().StringVarP( &options.zkAddr, "zk-addr", "z", "", "ZooKeeper address", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.zkPrefix, "zk-prefix", "", From 1b84ef384c12aa83494141102a4a47e135054ffb Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 15:17:15 -0400 Subject: [PATCH 011/116] remove unneeded variables --- cmd/topicctl/subcmd/get.go | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 67f8332b..33f7abfd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -64,7 +64,7 @@ func getPreRun(cmd *cobra.Command, args []string) error { } func balanceCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "balance [optional topic]", Short: "Number of replicas per broker position for topic or cluster as a whole", Long: strings.Join([]string{ @@ -94,11 +94,10 @@ func balanceCmd() *cobra.Command { }, PreRunE: getPreRun, } - return cmd } func brokersCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "brokers", Short: "Displays descriptions of each broker in the cluster.", Args: cobra.NoArgs, @@ -116,11 +115,10 @@ func brokersCmd() *cobra.Command { return cliRunner.GetBrokers(ctx, getConfig.full) }, } - return cmd } func configCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "config [broker or topic]", Short: "Displays configuration for the provider broker or topic.", Args: cobra.ExactArgs(1), @@ -138,11 +136,10 @@ func configCmd() *cobra.Command { return cliRunner.GetConfig(ctx, args[0]) }, } - return cmd } func groupsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "groups", Short: "Displays consumer group informatin for the cluster.", Args: cobra.NoArgs, @@ -160,11 +157,10 @@ func groupsCmd() *cobra.Command { return cliRunner.GetGroups(ctx) }, } - return cmd } func lagsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "lags [topic] [group]", Short: "Displays consumer group lag for the specified topic and consumer group.", Args: cobra.ExactArgs(2), @@ -188,11 +184,10 @@ func lagsCmd() *cobra.Command { ) }, } - return cmd } func membersCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "members [group]", Short: "Details of each member in the specified consumer group.", Args: cobra.ExactArgs(1), @@ -210,11 +205,10 @@ func membersCmd() *cobra.Command { return cliRunner.GetGroupMembers(ctx, args[0], getConfig.full) }, } - return cmd } func partitionsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "partitions [topic]", Short: "Displays partition information for the specified topic.", Args: cobra.ExactArgs(1), @@ -232,11 +226,10 @@ func partitionsCmd() *cobra.Command { return cliRunner.GetPartitions(ctx, args[0]) }, } - return cmd } func offsetsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "offsets [topic]", Short: "Displays offset information for the specified topic along with start and end times.", Args: cobra.ExactArgs(1), @@ -254,11 +247,10 @@ func offsetsCmd() *cobra.Command { return cliRunner.GetOffsets(ctx, args[0]) }, } - return cmd } func topicsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "topics", Short: "Displays information for all topics in the cluster.", Args: cobra.NoArgs, @@ -276,5 +268,4 @@ func topicsCmd() *cobra.Command { return cliRunner.GetTopics(ctx, getConfig.full) }, } - return cmd } From ea28ea99ac2dd1e55a69df0676000b1b907cffe7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 16:42:11 -0400 Subject: [PATCH 012/116] wip --- cmd/topicctl/subcmd/get.go | 23 +++++++++- pkg/admin/brokerclient.go | 2 +- pkg/admin/format.go | 2 +- pkg/admin/types.go | 90 +++++++++++++++++++++++++++++++++++++- 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index a33069ec..fb72c525 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -55,6 +55,7 @@ func init() { partitionsCmd(), offsetsCmd(), topicsCmd(), + aclsCmd(), ) RootCmd.AddCommand(getCmd) } @@ -267,6 +268,26 @@ func topicsCmd() *cobra.Command { cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetTopics(ctx, getConfig.full) }, ->>>>>>> chore/separate-subcmd-for-get + } +} + +func aclsCmd() *cobra.Command { + return &cobra.Command{ + Use: "acls", + Short: "Displays information for acls in the cluster. Supports filtering with flags.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetACLs(ctx) + }, } } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index fc34e000..9131e031 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -726,7 +726,7 @@ func (c *BrokerAdminClient) GetACLs( for _, resource := range resp.Resources { for _, acl := range resource.ACLs { aclinfos = append(aclinfos, ACLInfo{ - ResourceType: resource.ResourceType, + ResourceType: kafkaGoResourceTypeToTopicctl(resource.ResourceType), ResourceName: resource.ResourceName, PatternType: resource.PatternType, Principal: acl.Principal, diff --git a/pkg/admin/format.go b/pkg/admin/format.go index ec7e4645..87b1e963 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -784,7 +784,7 @@ func FormatAcls(acls []ACLInfo) string { for _, acl := range acls { row := []string{ // TODO: convert ints to something human readable - fmt.Sprintf("%d", acl.ResourceType), + string(acl.ResourceType), acl.ResourceName, acl.Principal, acl.Host, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 125f2160..13be6a9b 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -77,7 +77,7 @@ type PartitionAssignment struct { // PartitionInfo represents the information stored about an ACL // in zookeeper. type ACLInfo struct { - ResourceType kafka.ResourceType + ResourceType ACLResourceType ResourceName string PatternType kafka.PatternType Principal string @@ -86,6 +86,94 @@ type ACLInfo struct { PermissionType kafka.ACLPermissionType } +type ACLResourceType string + +func kafkaGoResourceTypeToTopicctl(r kafka.ResourceType) ACLResourceType { + switch r { + case kafka.ResourceTypeUnknown: + return "Unknown" + case kafka.ResourceTypeAny: + return "Any" + case kafka.ResourceTypeTopic: + return "Topic" + case kafka.ResourceTypeGroup: + return "Group" + case kafka.ResourceTypeCluster: + return "Cluster" + case kafka.ResourceTypeTransactionalID: + return "TransactionalID" + case kafka.ResourceTypeDelegationToken: + return "DelegationToken" + default: + return "Invalid ResourceType" + } +} + +// func (p kafka.PatternType) String() string { +// switch p { +// case kafka.PatternTypeUnknown: +// return "Unknown" +// case kafka.PatternTypeAny: +// return "Any" +// case kafka.PatternTypeMatch: +// return "Match" +// case kafka.PatternTypeLiteral: +// return "Literal" +// case kafka.PatternTypePrefixed: +// return "Prefixed" +// default: +// return "Invalid PatternType" +// } +// } + +// func (o kafka.ACLOperationType) String() string { +// switch o { +// case kafka.ACLOperationTypeUnknown: +// return "Unknown" +// case kafka.ACLOperationTypeAny: +// return "Any" +// case kafka.ACLOperationTypeAll: +// return "All" +// case kafka.ACLOperationTypeRead: +// return "Read" +// case kafka.ACLOperationTypeWrite: +// return "Write" +// case kafka.ACLOperationTypeCreate: +// return "Create" +// case kafka.ACLOperationTypeDelete: +// return "Delete" +// case kafka.ACLOperationTypeAlter: +// return "Alter" +// case kafka.ACLOperationTypeDescribe: +// return "Describe" +// case kafka.ACLOperationTypeClusterAction: +// return "ClusterAction" +// case kafka.ACLOperationTypeDescribeConfigs: +// return "DescribeConfigs" +// case kafka.ACLOperationTypeAlterConfigs: +// return "AlterConfigs" +// case kafka.ACLOperationTypeIdempotentWrite: +// return "IdempotentWrite" +// default: +// return "Invalid OperationType" +// } +// } + +// func (p kafka.ACLPermissionType) String() string { +// switch p { +// case kafka.ACLPermissionTypeUnknown: +// return "Unknown" +// case kafka.ACLPermissionTypeAny: +// return "Any" +// case kafka.ACLPermissionTypeDeny: +// return "Deny" +// case kafka.ACLPermissionTypeAllow: +// return "Allow" +// default: +// return "Invalid PermissionType" +// } +// } + type zkClusterID struct { Version string `json:"version"` ID string `json:"id"` From 07667ddf64bfbe9383cb2ae9f2f6b548f169f157 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 16:49:39 -0400 Subject: [PATCH 013/116] pr feedback --- cmd/topicctl/subcmd/get.go | 89 ++++++++++++-------------------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 33f7abfd..dd1a4c28 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -63,6 +63,24 @@ func getPreRun(cmd *cobra.Command, args []string) error { return getConfig.shared.validate() } +func getCliRunnerAndCtx() ( + context.Context, + *cli.CLIRunner, + error, +) { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return nil, nil, err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return ctx, cliRunner, nil +} + func balanceCmd() *cobra.Command { return &cobra.Command{ Use: "balance [optional topic]", @@ -75,16 +93,10 @@ func balanceCmd() *cobra.Command { ), Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) var topicName string if len(args) == 1 { @@ -102,16 +114,10 @@ func brokersCmd() *cobra.Command { Short: "Displays descriptions of each broker in the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetBrokers(ctx, getConfig.full) }, } @@ -123,16 +129,11 @@ func configCmd() *cobra.Command { Short: "Displays configuration for the provider broker or topic.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetConfig(ctx, args[0]) }, } @@ -144,16 +145,10 @@ func groupsCmd() *cobra.Command { Short: "Displays consumer group informatin for the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetGroups(ctx) }, } @@ -165,16 +160,10 @@ func lagsCmd() *cobra.Command { Short: "Displays consumer group lag for the specified topic and consumer group.", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetMemberLags( ctx, args[0], @@ -192,16 +181,10 @@ func membersCmd() *cobra.Command { Short: "Details of each member in the specified consumer group.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetGroupMembers(ctx, args[0], getConfig.full) }, } @@ -213,16 +196,10 @@ func partitionsCmd() *cobra.Command { Short: "Displays partition information for the specified topic.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetPartitions(ctx, args[0]) }, } @@ -234,16 +211,10 @@ func offsetsCmd() *cobra.Command { Short: "Displays offset information for the specified topic along with start and end times.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetOffsets(ctx, args[0]) }, } @@ -255,16 +226,10 @@ func topicsCmd() *cobra.Command { Short: "Displays information for all topics in the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetTopics(ctx, getConfig.full) }, } From dcdd0e8d3abd4988ac8b7a8db39467f6c22e5215 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 08:44:15 -0400 Subject: [PATCH 014/116] Revert "upgrade cobra to latest" This reverts commit 7b8ee4282d309441a9586db13dde05c5ec30eff5. --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0f232842..4293590e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/segmentio/kafka-go v0.4.35 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -23,7 +23,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/mattn/go-colorable v0.1.9 // indirect diff --git a/go.sum b/go.sum index b8fb8ecf..4e4f327e 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -80,8 +80,8 @@ github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d07 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 9f8f5507a41b21547814366c33f36949e105d6ec Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 09:08:59 -0400 Subject: [PATCH 015/116] use getCliRunnerAndCtx in get acls --- cmd/topicctl/subcmd/get.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index a1c32579..f8b57b79 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -242,16 +242,10 @@ func aclsCmd() *cobra.Command { Short: "Displays information for acls in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetACLs(ctx) }, } From 4a78af2c5b6b4632eb9154b0abff621bf2cbf5d0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 09:10:30 -0400 Subject: [PATCH 016/116] more consistent variable names --- pkg/admin/format.go | 4 ++-- pkg/cli/cli.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 87b1e963..5a3916ab 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -745,9 +745,9 @@ func FormatBrokerMaxPartitions( return string(bytes.TrimRight(buf.Bytes(), "\n")) } -// FormatAcls creates a pretty table that lists the details of the +// FormatACLs creates a pretty table that lists the details of the // argument acls. -func FormatAcls(acls []ACLInfo) string { +func FormatACLs(acls []ACLInfo) string { buf := &bytes.Buffer{} headers := []string{ diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 68223baa..c6de45ce 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -554,7 +554,7 @@ func (c *CLIRunner) Tail( } // TODO add options for filtering -// GetAcls fetches the details of each acl in the cluster and prints out a summary. +// GetACLs fetches the details of each acl in the cluster and prints out a summary. func (c *CLIRunner) GetACLs(ctx context.Context) error { c.startSpinner() @@ -569,7 +569,7 @@ func (c *CLIRunner) GetACLs(ctx context.Context) error { return err } - c.printer("ACLs:\n%s", admin.FormatAcls(acls)) + c.printer("ACLs:\n%s", admin.FormatACLs(acls)) return nil } From 1dbf20027e98233171465055bc4b9ea022dd2ca7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 11:23:37 -0400 Subject: [PATCH 017/116] custom cobra type --- cmd/topicctl/subcmd/get.go | 29 +++++++++++++++++--- pkg/admin/brokerclient.go | 2 +- pkg/admin/format.go | 2 +- pkg/admin/types.go | 55 +++++++++++++++++++++++++++++--------- pkg/cli/cli.go | 12 ++++----- 5 files changed, 76 insertions(+), 24 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index f8b57b79..6276fe50 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -5,6 +5,8 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws/session" + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -236,17 +238,38 @@ func topicsCmd() *cobra.Command { } } +type aclsCmdConfig struct { + resourceType admin.ResourceType +} + +var aclsConfig = aclsCmdConfig{ + resourceType: admin.ResourceType(kafka.ResourceTypeAny), +} + func aclsCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "acls", - Short: "Displays information for acls in the cluster. Supports filtering with flags.", + Short: "Displays information for ACLs in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - return cliRunner.GetACLs(ctx) + + filter := kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), + ResourcePatternTypeFilter: kafka.PatternTypeAny, + Operation: kafka.ACLOperationTypeAny, + PermissionType: kafka.ACLPermissionTypeAny, + } + return cliRunner.GetACLs(ctx, filter) }, } + cmd.Flags().Var( + &aclsConfig.resourceType, + "resource-type", + `Resource type. allowed: "unknown", "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + ) + return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 9131e031..dbe723df 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -726,7 +726,7 @@ func (c *BrokerAdminClient) GetACLs( for _, resource := range resp.Resources { for _, acl := range resource.ACLs { aclinfos = append(aclinfos, ACLInfo{ - ResourceType: kafkaGoResourceTypeToTopicctl(resource.ResourceType), + ResourceType: ResourceType(resource.ResourceType), ResourceName: resource.ResourceName, PatternType: resource.PatternType, Principal: acl.Principal, diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 5a3916ab..9c9d49bc 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -784,7 +784,7 @@ func FormatACLs(acls []ACLInfo) string { for _, acl := range acls { row := []string{ // TODO: convert ints to something human readable - string(acl.ResourceType), + acl.ResourceType.String(), acl.ResourceName, acl.Principal, acl.Host, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 13be6a9b..0599eb55 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -6,6 +6,7 @@ import ( "reflect" "sort" "strconv" + "strings" "time" "github.com/segmentio/kafka-go" @@ -77,7 +78,7 @@ type PartitionAssignment struct { // PartitionInfo represents the information stored about an ACL // in zookeeper. type ACLInfo struct { - ResourceType ACLResourceType + ResourceType ResourceType ResourceName string PatternType kafka.PatternType Principal string @@ -86,29 +87,59 @@ type ACLInfo struct { PermissionType kafka.ACLPermissionType } -type ACLResourceType string +// ResourceType presents the Kafka resource type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type ResourceType kafka.ResourceType -func kafkaGoResourceTypeToTopicctl(r kafka.ResourceType) ACLResourceType { - switch r { +var resourceTypeMap = map[string]kafka.ResourceType{ + "unknown": kafka.ResourceTypeUnknown, + "any": kafka.ResourceTypeAny, + "topic": kafka.ResourceTypeTopic, + "group": kafka.ResourceTypeGroup, + "cluster": kafka.ResourceTypeCluster, + "transactionalid": kafka.ResourceTypeTransactionalID, + "delegationtoken": kafka.ResourceTypeDelegationToken, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (r *ResourceType) String() string { + switch kafka.ResourceType(*r) { case kafka.ResourceTypeUnknown: - return "Unknown" + return "unknown" case kafka.ResourceTypeAny: - return "Any" + return "any" case kafka.ResourceTypeTopic: - return "Topic" + return "topic" case kafka.ResourceTypeGroup: - return "Group" + return "group" case kafka.ResourceTypeCluster: - return "Cluster" + return "cluster" case kafka.ResourceTypeTransactionalID: - return "TransactionalID" + return "transactionalid" case kafka.ResourceTypeDelegationToken: - return "DelegationToken" + return "delegationtoken" default: - return "Invalid ResourceType" + return "invalid ResourceType" } } +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (r *ResourceType) Set(v string) error { + rt, ok := resourceTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "unknown", "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) + } + *r = ResourceType(rt) + return nil +} + +// Type is used by Cobra in help text. +func (r *ResourceType) Type() string { + return "ResourceType" +} + // func (p kafka.PatternType) String() string { // switch p { // case kafka.PatternTypeUnknown: diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index c6de45ce..40e4f901 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -555,15 +555,13 @@ func (c *CLIRunner) Tail( // TODO add options for filtering // GetACLs fetches the details of each acl in the cluster and prints out a summary. -func (c *CLIRunner) GetACLs(ctx context.Context) error { +func (c *CLIRunner) GetACLs( + ctx context.Context, + filter kafka.ACLFilter, +) error { c.startSpinner() - acls, err := c.adminClient.GetACLs(ctx, kafka.ACLFilter{ - ResourceTypeFilter: kafka.ResourceTypeAny, - ResourcePatternTypeFilter: kafka.PatternTypeAny, - Operation: kafka.ACLOperationTypeAny, - PermissionType: kafka.ACLPermissionTypeAny, - }) + acls, err := c.adminClient.GetACLs(ctx, filter) c.stopSpinner() if err != nil { return err From 226ae1c27dec4ade687d07159acd05dc830d4cc4 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 13:01:03 -0400 Subject: [PATCH 018/116] bring in new kafka-go --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7f969696..66a35960 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 + github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 4d6c3b80..6e87f76d 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,12 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= +github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= +github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= +github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 h1:Gok6q1P5yUVLYt+auyZgcRBP89thqCmU9MNT65Ms1SI= +github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= From acc011f144e50e58f4c0e952838e4569f32b54b0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 15:36:44 -0400 Subject: [PATCH 019/116] support resource pattern type --- cmd/topicctl/subcmd/get.go | 13 +++++-- pkg/admin/brokerclient.go | 2 +- pkg/admin/brokerclient_test.go | 31 +++++++++++----- pkg/admin/format.go | 3 ++ pkg/admin/types.go | 64 +++++++++++++++++++++++++--------- 5 files changed, 83 insertions(+), 30 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 6276fe50..acda69d1 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -239,11 +239,13 @@ func topicsCmd() *cobra.Command { } type aclsCmdConfig struct { - resourceType admin.ResourceType + resourceType admin.ResourceType + resourcePatternType admin.PatternType } var aclsConfig = aclsCmdConfig{ - resourceType: admin.ResourceType(kafka.ResourceTypeAny), + resourceType: admin.ResourceType(kafka.ResourceTypeAny), + resourcePatternType: admin.PatternType(kafka.PatternTypeAny), } func aclsCmd() *cobra.Command { @@ -259,7 +261,7 @@ func aclsCmd() *cobra.Command { filter := kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), - ResourcePatternTypeFilter: kafka.PatternTypeAny, + ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), Operation: kafka.ACLOperationTypeAny, PermissionType: kafka.ACLPermissionTypeAny, } @@ -271,5 +273,10 @@ func aclsCmd() *cobra.Command { "resource-type", `Resource type. allowed: "unknown", "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, ) + cmd.Flags().Var( + &aclsConfig.resourcePatternType, + "resource-pattern-type", + `Resource pattern type. allowed: "unknown", "any", "match", "literal", "prefixed"`, + ) return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index dbe723df..1bb7e22f 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -728,7 +728,7 @@ func (c *BrokerAdminClient) GetACLs( aclinfos = append(aclinfos, ACLInfo{ ResourceType: ResourceType(resource.ResourceType), ResourceName: resource.ResourceName, - PatternType: resource.PatternType, + PatternType: PatternType(resource.PatternType), Principal: acl.Principal, Host: acl.Host, Operation: acl.Operation, diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 2d8c21b6..f849fc6b 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -586,13 +586,6 @@ func TestBrokerClientCreateGetACL(t *testing.T) { BrokerAdminClientConfig{ ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), - SASL: SASLConfig{ - Enabled: true, - Mechanism: SASLMechanismScramSHA512, - // TODO: don't hardcode these in tests, pull from env vars - Username: "adminscram", - Password: "admin-secret-512", - }, }, }, ) @@ -601,6 +594,27 @@ func TestBrokerClientCreateGetACL(t *testing.T) { principal := util.RandomString("User:user-create-", 6) topicName := util.RandomString("topic-create-", 6) + defer func() { + _, err := client.client.DeleteACLs( + ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + err = client.CreateACL( ctx, kafka.ACLEntry{ @@ -627,7 +641,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { require.NoError(t, err) expected := []ACLInfo{ { - ResourceType: kafka.ResourceTypeTopic, + ResourceType: ResourceType(kafka.ResourceTypeTopic), ResourceName: topicName, PatternType: kafka.PatternTypeLiteral, Principal: principal, @@ -637,5 +651,4 @@ func TestBrokerClientCreateGetACL(t *testing.T) { }, } assert.Equal(t, expected, aclsInfo) - } diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 9c9d49bc..2347d1c2 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -752,6 +752,7 @@ func FormatACLs(acls []ACLInfo) string { headers := []string{ "Resource Type", + "Pattern Type", "Resource Name", "Principal", "Host", @@ -770,6 +771,7 @@ func FormatACLs(acls []ACLInfo) string { tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, }, ) table.SetBorders( @@ -785,6 +787,7 @@ func FormatACLs(acls []ACLInfo) string { row := []string{ // TODO: convert ints to something human readable acl.ResourceType.String(), + acl.PatternType.String(), acl.ResourceName, acl.Principal, acl.Host, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 0599eb55..3fd427b8 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -80,7 +80,7 @@ type PartitionAssignment struct { type ACLInfo struct { ResourceType ResourceType ResourceName string - PatternType kafka.PatternType + PatternType PatternType Principal string Host string Operation kafka.ACLOperationType @@ -140,22 +140,52 @@ func (r *ResourceType) Type() string { return "ResourceType" } -// func (p kafka.PatternType) String() string { -// switch p { -// case kafka.PatternTypeUnknown: -// return "Unknown" -// case kafka.PatternTypeAny: -// return "Any" -// case kafka.PatternTypeMatch: -// return "Match" -// case kafka.PatternTypeLiteral: -// return "Literal" -// case kafka.PatternTypePrefixed: -// return "Prefixed" -// default: -// return "Invalid PatternType" -// } -// } +// PatternType presents the Kafka resource type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type PatternType kafka.PatternType + +var patternTypeMap = map[string]kafka.PatternType{ + "unknown": kafka.PatternTypeUnknown, + "any": kafka.PatternTypeAny, + "match": kafka.PatternTypeMatch, + "literal": kafka.PatternTypeLiteral, + "prefixed": kafka.PatternTypePrefixed, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (p *PatternType) String() string { + switch kafka.PatternType(*p) { + case kafka.PatternTypeUnknown: + return "unknown" + case kafka.PatternTypeAny: + return "any" + case kafka.PatternTypeMatch: + return "match" + case kafka.PatternTypeLiteral: + return "literal" + case kafka.PatternTypePrefixed: + return "prefixed" + default: + return "invalid PatternType" + } +} + +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (r *PatternType) Set(v string) error { + rt, ok := patternTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "unknown", "any", "match", "literal", or "prefixed"`) + } + *r = PatternType(rt) + return nil +} + +// Type is used by Cobra in help text. +func (r *PatternType) Type() string { + return "PatternType" +} // func (o kafka.ACLOperationType) String() string { // switch o { From 2536e506fa11d3c79fade5dc3ba7b8b976ab78e1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 10:25:07 -0400 Subject: [PATCH 020/116] add support for acloperationtype and remove options for unknown --- cmd/topicctl/subcmd/get.go | 17 +++++-- pkg/admin/brokerclient.go | 2 +- pkg/admin/brokerclient_test.go | 4 +- pkg/admin/format.go | 2 +- pkg/admin/types.go | 84 +++++++++++++++++++++++++++++----- 5 files changed, 91 insertions(+), 18 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index acda69d1..9df654cd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -241,11 +241,13 @@ func topicsCmd() *cobra.Command { type aclsCmdConfig struct { resourceType admin.ResourceType resourcePatternType admin.PatternType + aclOperationType admin.ACLOperationType } var aclsConfig = aclsCmdConfig{ resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), + aclOperationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), } func aclsCmd() *cobra.Command { @@ -253,6 +255,8 @@ func aclsCmd() *cobra.Command { Use: "acls", Short: "Displays information for ACLs in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, + // TODO: make common examples here + // Example: RunE: func(cmd *cobra.Command, args []string) error { ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { @@ -260,9 +264,10 @@ func aclsCmd() *cobra.Command { } filter := kafka.ACLFilter{ + //ResourceNameFilter: "", ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), - Operation: kafka.ACLOperationTypeAny, + Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), PermissionType: kafka.ACLPermissionTypeAny, } return cliRunner.GetACLs(ctx, filter) @@ -271,12 +276,18 @@ func aclsCmd() *cobra.Command { cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", - `Resource type. allowed: "unknown", "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + `Resource type. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", - `Resource pattern type. allowed: "unknown", "any", "match", "literal", "prefixed"`, + // TODO: document the behavior of each of these + `Resource pattern type. allowed: "any", "match", "literal", "prefixed"`, + ) + cmd.Flags().Var( + &aclsConfig.aclOperationType, + "operations", + `ACL operation type. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, ) return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 1bb7e22f..c6910c41 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -731,7 +731,7 @@ func (c *BrokerAdminClient) GetACLs( PatternType: PatternType(resource.PatternType), Principal: acl.Principal, Host: acl.Host, - Operation: acl.Operation, + Operation: ACLOperationType(acl.Operation), PermissionType: acl.PermissionType, }) } diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index f849fc6b..9a9c77c3 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -643,10 +643,10 @@ func TestBrokerClientCreateGetACL(t *testing.T) { { ResourceType: ResourceType(kafka.ResourceTypeTopic), ResourceName: topicName, - PatternType: kafka.PatternTypeLiteral, + PatternType: PatternType(kafka.PatternTypeLiteral), Principal: principal, Host: "*", - Operation: kafka.ACLOperationTypeRead, + Operation: ACLOperationType(kafka.ACLOperationTypeRead), PermissionType: kafka.ACLPermissionTypeAllow, }, } diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 2347d1c2..6cfa33aa 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -791,7 +791,7 @@ func FormatACLs(acls []ACLInfo) string { acl.ResourceName, acl.Principal, acl.Host, - fmt.Sprintf("%d", acl.Operation), + acl.Operation.String(), fmt.Sprintf("%d", acl.PermissionType), } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 3fd427b8..a336bcfe 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -83,7 +83,7 @@ type ACLInfo struct { PatternType PatternType Principal string Host string - Operation kafka.ACLOperationType + Operation ACLOperationType PermissionType kafka.ACLPermissionType } @@ -94,7 +94,6 @@ type ACLInfo struct { type ResourceType kafka.ResourceType var resourceTypeMap = map[string]kafka.ResourceType{ - "unknown": kafka.ResourceTypeUnknown, "any": kafka.ResourceTypeAny, "topic": kafka.ResourceTypeTopic, "group": kafka.ResourceTypeGroup, @@ -106,8 +105,6 @@ var resourceTypeMap = map[string]kafka.ResourceType{ // String is used both by fmt.Print and by Cobra in help text. func (r *ResourceType) String() string { switch kafka.ResourceType(*r) { - case kafka.ResourceTypeUnknown: - return "unknown" case kafka.ResourceTypeAny: return "any" case kafka.ResourceTypeTopic: @@ -121,7 +118,7 @@ func (r *ResourceType) String() string { case kafka.ResourceTypeDelegationToken: return "delegationtoken" default: - return "invalid ResourceType" + return "unknown" } } @@ -129,7 +126,7 @@ func (r *ResourceType) String() string { func (r *ResourceType) Set(v string) error { rt, ok := resourceTypeMap[strings.ToLower(v)] if !ok { - return errors.New(`must be one of "unknown", "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) + return errors.New(`must be one of "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) } *r = ResourceType(rt) return nil @@ -147,7 +144,6 @@ func (r *ResourceType) Type() string { type PatternType kafka.PatternType var patternTypeMap = map[string]kafka.PatternType{ - "unknown": kafka.PatternTypeUnknown, "any": kafka.PatternTypeAny, "match": kafka.PatternTypeMatch, "literal": kafka.PatternTypeLiteral, @@ -157,8 +153,6 @@ var patternTypeMap = map[string]kafka.PatternType{ // String is used both by fmt.Print and by Cobra in help text. func (p *PatternType) String() string { switch kafka.PatternType(*p) { - case kafka.PatternTypeUnknown: - return "unknown" case kafka.PatternTypeAny: return "any" case kafka.PatternTypeMatch: @@ -168,7 +162,7 @@ func (p *PatternType) String() string { case kafka.PatternTypePrefixed: return "prefixed" default: - return "invalid PatternType" + return "unknown" } } @@ -176,7 +170,7 @@ func (p *PatternType) String() string { func (r *PatternType) Set(v string) error { rt, ok := patternTypeMap[strings.ToLower(v)] if !ok { - return errors.New(`must be one of "unknown", "any", "match", "literal", or "prefixed"`) + return errors.New(`must be one of "any", "match", "literal", or "prefixed"`) } *r = PatternType(rt) return nil @@ -187,6 +181,74 @@ func (r *PatternType) Type() string { return "PatternType" } +// ACLOperationType presents the Kafka resource type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type ACLOperationType kafka.ACLOperationType + +var aclOperationTypeMap = map[string]kafka.ACLOperationType{ + "any": kafka.ACLOperationTypeAny, + "all": kafka.ACLOperationTypeAll, + "read": kafka.ACLOperationTypeRead, + "write": kafka.ACLOperationTypeWrite, + "create": kafka.ACLOperationTypeCreate, + "delete": kafka.ACLOperationTypeDelete, + "alter": kafka.ACLOperationTypeAlter, + "describe": kafka.ACLOperationTypeDescribe, + "clusteraction": kafka.ACLOperationTypeClusterAction, + "describeconfigs": kafka.ACLOperationTypeDescribeConfigs, + "alterconfigs": kafka.ACLOperationTypeAlterConfigs, + "idempotentwrite": kafka.ACLOperationTypeIdempotentWrite, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (o *ACLOperationType) String() string { + switch kafka.ACLOperationType(*o) { + case kafka.ACLOperationTypeAny: + return "any" + case kafka.ACLOperationTypeAll: + return "all" + case kafka.ACLOperationTypeRead: + return "read" + case kafka.ACLOperationTypeWrite: + return "write" + case kafka.ACLOperationTypeCreate: + return "create" + case kafka.ACLOperationTypeDelete: + return "delete" + case kafka.ACLOperationTypeAlter: + return "alter" + case kafka.ACLOperationTypeDescribe: + return "describe" + case kafka.ACLOperationTypeClusterAction: + return "clusteraction" + case kafka.ACLOperationTypeDescribeConfigs: + return "describeconfigs" + case kafka.ACLOperationTypeAlterConfigs: + return "alterconfigs" + case kafka.ACLOperationTypeIdempotentWrite: + return "idempotentwrite" + default: + return "unknown" + } +} + +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (r *ACLOperationType) Set(v string) error { + rt, ok := aclOperationTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`) + } + *r = ACLOperationType(rt) + return nil +} + +// Type is used by Cobra in help text. +func (r *ACLOperationType) Type() string { + return "ACLOperationType" +} + // func (o kafka.ACLOperationType) String() string { // switch o { // case kafka.ACLOperationTypeUnknown: From 62671b0b960411e1ae81532d06cf9ffe8ddc55e3 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 10:55:31 -0400 Subject: [PATCH 021/116] improve descriptions --- cmd/topicctl/subcmd/get.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 9df654cd..e94ca3db 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -264,11 +264,13 @@ func aclsCmd() *cobra.Command { } filter := kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), //ResourceNameFilter: "", - ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), - Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), - PermissionType: kafka.ACLPermissionTypeAny, + //PrincipalFilter: "*", + //HostFilter:"*". + Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), + PermissionType: kafka.ACLPermissionTypeAny, } return cliRunner.GetACLs(ctx, filter) }, @@ -276,18 +278,19 @@ func aclsCmd() *cobra.Command { cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", - `Resource type. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", // TODO: document the behavior of each of these - `Resource pattern type. allowed: "any", "match", "literal", "prefixed"`, + // TODO: match isn't really supported right now, look into that + `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, ) cmd.Flags().Var( &aclsConfig.aclOperationType, "operations", - `ACL operation type. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, + `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, ) return cmd } From 3f050ca4e9eb26c3d1faf551628412fff82a148b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 11:17:26 -0400 Subject: [PATCH 022/116] support permissiontype and host filters --- cmd/topicctl/subcmd/get.go | 37 +++++++++---- pkg/admin/brokerclient.go | 2 +- pkg/admin/types.go | 107 +++++++++++++++++-------------------- 3 files changed, 78 insertions(+), 68 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index e94ca3db..2d5ed595 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -239,15 +239,21 @@ func topicsCmd() *cobra.Command { } type aclsCmdConfig struct { + hostFilter string + operationType admin.ACLOperationType + permissionType admin.ACLPermissionType resourceType admin.ResourceType resourcePatternType admin.PatternType - aclOperationType admin.ACLOperationType } +// aclsConfig defines the default values if a flag is not provided. These all default +// to doing no filtering (e.g. "all") var aclsConfig = aclsCmdConfig{ + hostFilter: "", + operationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), + permissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAny), resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), - aclOperationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), } func aclsCmd() *cobra.Command { @@ -268,13 +274,29 @@ func aclsCmd() *cobra.Command { //ResourceNameFilter: "", ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), //PrincipalFilter: "*", - //HostFilter:"*". - Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), - PermissionType: kafka.ACLPermissionTypeAny, + HostFilter: aclsConfig.hostFilter, + Operation: kafka.ACLOperationType(aclsConfig.operationType), + PermissionType: kafka.ACLPermissionType(aclsConfig.permissionType), } return cliRunner.GetACLs(ctx, filter) }, } + cmd.Flags().StringVar( + &aclsConfig.hostFilter, + "host", + "", + `The host to filter on.`, + ) + cmd.Flags().Var( + &aclsConfig.operationType, + "operations", + `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, + ) + cmd.Flags().Var( + &aclsConfig.permissionType, + "permission-type", + `The permission type to filter on. allowed: "any", "allow", or "deny"`, + ) cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", @@ -287,10 +309,5 @@ func aclsCmd() *cobra.Command { // TODO: match isn't really supported right now, look into that `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, ) - cmd.Flags().Var( - &aclsConfig.aclOperationType, - "operations", - `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, - ) return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index c6910c41..329fa1b3 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -732,7 +732,7 @@ func (c *BrokerAdminClient) GetACLs( Principal: acl.Principal, Host: acl.Host, Operation: ACLOperationType(acl.Operation), - PermissionType: acl.PermissionType, + PermissionType: ACLPermissionType(acl.PermissionType), }) } } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index a336bcfe..14b7c744 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -84,7 +84,7 @@ type ACLInfo struct { Principal string Host string Operation ACLOperationType - PermissionType kafka.ACLPermissionType + PermissionType ACLPermissionType } // ResourceType presents the Kafka resource type. @@ -137,7 +137,7 @@ func (r *ResourceType) Type() string { return "ResourceType" } -// PatternType presents the Kafka resource type. +// PatternType presents the Kafka pattern type. // We need to subtype this to be able to define methods to // satisfy the Value interface from Cobra so we can use it // as a Cobra flag. @@ -167,12 +167,12 @@ func (p *PatternType) String() string { } // Set is used by Cobra to set the value of a variable from a Cobra flag. -func (r *PatternType) Set(v string) error { - rt, ok := patternTypeMap[strings.ToLower(v)] +func (p *PatternType) Set(v string) error { + pt, ok := patternTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "match", "literal", or "prefixed"`) } - *r = PatternType(rt) + *p = PatternType(pt) return nil } @@ -181,7 +181,7 @@ func (r *PatternType) Type() string { return "PatternType" } -// ACLOperationType presents the Kafka resource type. +// ACLOperationType presents the Kafka operation type. // We need to subtype this to be able to define methods to // satisfy the Value interface from Cobra so we can use it // as a Cobra flag. @@ -235,67 +235,60 @@ func (o *ACLOperationType) String() string { } // Set is used by Cobra to set the value of a variable from a Cobra flag. -func (r *ACLOperationType) Set(v string) error { - rt, ok := aclOperationTypeMap[strings.ToLower(v)] +func (o *ACLOperationType) Set(v string) error { + ot, ok := aclOperationTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`) } - *r = ACLOperationType(rt) + *o = ACLOperationType(ot) return nil } // Type is used by Cobra in help text. -func (r *ACLOperationType) Type() string { +func (o *ACLOperationType) Type() string { return "ACLOperationType" } -// func (o kafka.ACLOperationType) String() string { -// switch o { -// case kafka.ACLOperationTypeUnknown: -// return "Unknown" -// case kafka.ACLOperationTypeAny: -// return "Any" -// case kafka.ACLOperationTypeAll: -// return "All" -// case kafka.ACLOperationTypeRead: -// return "Read" -// case kafka.ACLOperationTypeWrite: -// return "Write" -// case kafka.ACLOperationTypeCreate: -// return "Create" -// case kafka.ACLOperationTypeDelete: -// return "Delete" -// case kafka.ACLOperationTypeAlter: -// return "Alter" -// case kafka.ACLOperationTypeDescribe: -// return "Describe" -// case kafka.ACLOperationTypeClusterAction: -// return "ClusterAction" -// case kafka.ACLOperationTypeDescribeConfigs: -// return "DescribeConfigs" -// case kafka.ACLOperationTypeAlterConfigs: -// return "AlterConfigs" -// case kafka.ACLOperationTypeIdempotentWrite: -// return "IdempotentWrite" -// default: -// return "Invalid OperationType" -// } -// } - -// func (p kafka.ACLPermissionType) String() string { -// switch p { -// case kafka.ACLPermissionTypeUnknown: -// return "Unknown" -// case kafka.ACLPermissionTypeAny: -// return "Any" -// case kafka.ACLPermissionTypeDeny: -// return "Deny" -// case kafka.ACLPermissionTypeAllow: -// return "Allow" -// default: -// return "Invalid PermissionType" -// } -// } +// ACLPermissionType presents the Kafka operation type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type ACLPermissionType kafka.ACLPermissionType + +var aclPermissionTypeMap = map[string]kafka.ACLPermissionType{ + "any": kafka.ACLPermissionTypeAny, + "allow": kafka.ACLPermissionTypeAllow, + "deny": kafka.ACLPermissionTypeDeny, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (p *ACLPermissionType) String() string { + switch kafka.ACLPermissionType(*p) { + case kafka.ACLPermissionTypeAny: + return "any" + case kafka.ACLPermissionTypeDeny: + return "deny" + case kafka.ACLPermissionTypeAllow: + return "allow" + default: + return "unknown" + } +} + +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (p *ACLPermissionType) Set(v string) error { + pt, ok := aclPermissionTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "any", "allow", or "deny"`) + } + *p = ACLPermissionType(pt) + return nil +} + +// Type is used by Cobra in help text. +func (p *ACLPermissionType) Type() string { + return "ACLPermissionType" +} type zkClusterID struct { Version string `json:"version"` From 925670ce87e257fd22671f5e3da64ba99221ba53 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 13:37:31 -0400 Subject: [PATCH 023/116] add resource name filter and fix permission type formatting --- cmd/topicctl/subcmd/get.go | 22 +++++++++++++++------- pkg/admin/format.go | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 2d5ed595..b8163076 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -242,8 +242,9 @@ type aclsCmdConfig struct { hostFilter string operationType admin.ACLOperationType permissionType admin.ACLPermissionType - resourceType admin.ResourceType + resourceNameFilter string resourcePatternType admin.PatternType + resourceType admin.ResourceType } // aclsConfig defines the default values if a flag is not provided. These all default @@ -252,6 +253,7 @@ var aclsConfig = aclsCmdConfig{ hostFilter: "", operationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), permissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAny), + resourceNameFilter: "", resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), } @@ -270,8 +272,8 @@ func aclsCmd() *cobra.Command { } filter := kafka.ACLFilter{ - ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), - //ResourceNameFilter: "", + ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), + ResourceNameFilter: aclsConfig.resourceNameFilter, ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), //PrincipalFilter: "*", HostFilter: aclsConfig.hostFilter, @@ -297,10 +299,11 @@ func aclsCmd() *cobra.Command { "permission-type", `The permission type to filter on. allowed: "any", "allow", or "deny"`, ) - cmd.Flags().Var( - &aclsConfig.resourceType, - "resource-type", - `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + cmd.Flags().StringVar( + &aclsConfig.resourceNameFilter, + "resource-name", + "", + `The resource name to filter on.`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, @@ -309,5 +312,10 @@ func aclsCmd() *cobra.Command { // TODO: match isn't really supported right now, look into that `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, ) + cmd.Flags().Var( + &aclsConfig.resourceType, + "resource-type", + `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + ) return cmd } diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 6cfa33aa..1173a7ea 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -792,7 +792,7 @@ func FormatACLs(acls []ACLInfo) string { acl.Principal, acl.Host, acl.Operation.String(), - fmt.Sprintf("%d", acl.PermissionType), + acl.PermissionType.String(), } table.Append(row) From 5cff3325d1e20f7ae0747d27721e50f2cf3442ac Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 13:41:26 -0400 Subject: [PATCH 024/116] support principal filtering --- cmd/topicctl/subcmd/get.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index b8163076..7b7b731c 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -242,17 +242,19 @@ type aclsCmdConfig struct { hostFilter string operationType admin.ACLOperationType permissionType admin.ACLPermissionType + principalFilter string resourceNameFilter string resourcePatternType admin.PatternType resourceType admin.ResourceType } // aclsConfig defines the default values if a flag is not provided. These all default -// to doing no filtering (e.g. "all") +// to doing no filtering (e.g. "all" or null) var aclsConfig = aclsCmdConfig{ hostFilter: "", operationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), permissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAny), + principalFilter: "", resourceNameFilter: "", resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), @@ -275,10 +277,10 @@ func aclsCmd() *cobra.Command { ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), ResourceNameFilter: aclsConfig.resourceNameFilter, ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), - //PrincipalFilter: "*", - HostFilter: aclsConfig.hostFilter, - Operation: kafka.ACLOperationType(aclsConfig.operationType), - PermissionType: kafka.ACLPermissionType(aclsConfig.permissionType), + PrincipalFilter: aclsConfig.principalFilter, + HostFilter: aclsConfig.hostFilter, + Operation: kafka.ACLOperationType(aclsConfig.operationType), + PermissionType: kafka.ACLPermissionType(aclsConfig.permissionType), } return cliRunner.GetACLs(ctx, filter) }, @@ -299,6 +301,12 @@ func aclsCmd() *cobra.Command { "permission-type", `The permission type to filter on. allowed: "any", "allow", or "deny"`, ) + cmd.Flags().StringVar( + &aclsConfig.principalFilter, + "principal", + "", + `The principal to filter on in principalType:name format (e.g. User:alice).`, + ) cmd.Flags().StringVar( &aclsConfig.resourceNameFilter, "resource-name", From e6e8c634a44b4206390887f935dfe64382ae2e33 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 14:17:26 -0400 Subject: [PATCH 025/116] improve docs --- cmd/topicctl/subcmd/get.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 7b7b731c..4deb0026 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -289,7 +289,7 @@ func aclsCmd() *cobra.Command { &aclsConfig.hostFilter, "host", "", - `The host to filter on.`, + `The host to filter on. (e.g. 198.51.100.0)`, ) cmd.Flags().Var( &aclsConfig.operationType, @@ -311,14 +311,14 @@ func aclsCmd() *cobra.Command { &aclsConfig.resourceNameFilter, "resource-name", "", - `The resource name to filter on.`, + `The resource name to filter on. (e.g. my-topic)`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", // TODO: document the behavior of each of these // TODO: match isn't really supported right now, look into that - `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, + `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed". "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, ) cmd.Flags().Var( &aclsConfig.resourceType, From e28cb01d9be4d86e56aef4eb41b0bfe3a14ab83c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 10:58:13 -0400 Subject: [PATCH 026/116] add examples --- cmd/topicctl/subcmd/get.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 4deb0026..9d810ffd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -265,8 +265,18 @@ func aclsCmd() *cobra.Command { Use: "acls", Short: "Displays information for ACLs in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, - // TODO: make common examples here - // Example: + Example: `List all acls +$ topicctl get acls + +List read acls for topic my-topic +$ topicctl get acls --resource-type topic --resource-name my-topic --operations read + +List acls for user Alice with permission allow +$ topicctl get acls --principal User:alice --permission-type allow + +List acls for host 198.51.100.0 +$ topicctl get acls --host 198.51.100.0 +`, RunE: func(cmd *cobra.Command, args []string) error { ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { @@ -291,6 +301,7 @@ func aclsCmd() *cobra.Command { "", `The host to filter on. (e.g. 198.51.100.0)`, ) + // TODO: support multiple comma separated ones cmd.Flags().Var( &aclsConfig.operationType, "operations", @@ -316,8 +327,6 @@ func aclsCmd() *cobra.Command { cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", - // TODO: document the behavior of each of these - // TODO: match isn't really supported right now, look into that `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed". "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, ) cmd.Flags().Var( From 9735b1b104b3a895c4386acca51c93137adbc476 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:07:10 -0400 Subject: [PATCH 027/116] remove comment --- cmd/topicctl/subcmd/get.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 9d810ffd..5461397a 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -301,7 +301,6 @@ $ topicctl get acls --host 198.51.100.0 "", `The host to filter on. (e.g. 198.51.100.0)`, ) - // TODO: support multiple comma separated ones cmd.Flags().Var( &aclsConfig.operationType, "operations", From b19a4e145052593a466a48000fb678cd72010483 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:08:05 -0400 Subject: [PATCH 028/116] remove TODOs that are complete --- pkg/admin/format.go | 1 - pkg/cli/cli.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 1173a7ea..fe0625d0 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -785,7 +785,6 @@ func FormatACLs(acls []ACLInfo) string { for _, acl := range acls { row := []string{ - // TODO: convert ints to something human readable acl.ResourceType.String(), acl.PatternType.String(), acl.ResourceName, diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 40e4f901..84b37bd6 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -553,7 +553,6 @@ func (c *CLIRunner) Tail( return err } -// TODO add options for filtering // GetACLs fetches the details of each acl in the cluster and prints out a summary. func (c *CLIRunner) GetACLs( ctx context.Context, From 43806c0a86a2fbb1a8ae79e90dd9f94b50bf8e5c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:08:41 -0400 Subject: [PATCH 029/116] remove TODOs that are complete --- pkg/admin/brokerclient.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 329fa1b3..9fbf5e7f 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -697,10 +697,6 @@ func configEntriesToAPIConfigs( return apiConfigs } -// TODO: what fields should we let people filter on / how best to support that? -// It could be really useful to be able to use this to answer questions like what services have access topics x,y,z or -// who has write access to topic b? -// Is GetACL (single ACL) even applicable? // GetACLs gets full information about each ACL in the cluster. func (c *BrokerAdminClient) GetACLs( ctx context.Context, From 2fb8c8e531883df80b6650777675c04a4199e7f1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:15:12 -0400 Subject: [PATCH 030/116] update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4ae37db1..5701dbff 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ resource type in the cluster. Currently, the following operations are supported: | `get partitions [topic]` | All partitions in a topic | | `get offsets [topic]` | Number of messages per partition along with start and end times | | `get topics` | All topics in the cluster | +| `get acls [flags]` | Describe access control levels (ACLs) in the cluster | #### rebalance From 45d403d8728ecd03a7a6784847d25c147c715794 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:16:26 -0400 Subject: [PATCH 031/116] fix test --- pkg/admin/brokerclient_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 9a9c77c3..66f25ab6 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -647,7 +647,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { Principal: principal, Host: "*", Operation: ACLOperationType(kafka.ACLOperationTypeRead), - PermissionType: kafka.ACLPermissionTypeAllow, + PermissionType: ACLPermissionType(kafka.ACLPermissionTypeAllow), }, } assert.Equal(t, expected, aclsInfo) From b3a5ef89cd2907a9b1fd2e40baf8238c889f5e20 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 12:04:58 -0400 Subject: [PATCH 032/116] wip --- cmd/topicctl/subcmd/get.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 33d34fed..5461397a 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -208,7 +208,6 @@ func partitionsCmd() *cobra.Command { } } ->>>>>>> master func offsetsCmd() *cobra.Command { return &cobra.Command{ Use: "offsets [topic]", From 6c1f7f11ddafc58b5e30d97321ac1015e62b8d13 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 12:08:01 -0400 Subject: [PATCH 033/116] fix error handling --- pkg/admin/brokerclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 9fbf5e7f..14b73e26 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -758,7 +758,7 @@ func (c *BrokerAdminClient) CreateACL( return err } if len(resp.Errors) > 0 { - fmt.Errorf("%+v", resp.Errors) + return fmt.Errorf("%+v", resp.Errors) } return nil } From cd3a1f66339f9a788fcf6cc93cb3a07a2b9905fe Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:16:16 -0400 Subject: [PATCH 034/116] error handling for zk --- pkg/admin/zkclient.go | 2 +- pkg/admin/zkclient_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index fa032764..13e604cb 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -426,7 +426,7 @@ func (c *ZKAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, ) ([]ACLInfo, error) { - return nil, nil + return nil, errors.New("Zookeeper client does not yet support GetACLs. Remove zk-addr and use the broker client instead") } // UpdateTopicConfig updates the config JSON for a topic and sets a change diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 210b23fb..d2adf21d 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1074,3 +1074,20 @@ func TestZkClientLocking(t *testing.T) { func testClusterID(name string) string { return util.RandomString(fmt.Sprintf("cluster-%s-", name), 6) } + +func TestZkGetACLs(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + BootstrapAddrs: []string{util.TestKafkaAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + acls, err := adminClient.GetACLs(ctx, kafka.ACLFilter{}) + assert.Empty(t, acls) + assert.Error(t, err) +} From e0c8c635af965059f90311c32b8ea208300dcca9 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:20:27 -0400 Subject: [PATCH 035/116] more consistent error msg --- pkg/admin/zkclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 13e604cb..5a6f115e 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -426,7 +426,7 @@ func (c *ZKAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, ) ([]ACLInfo, error) { - return nil, errors.New("Zookeeper client does not yet support GetACLs. Remove zk-addr and use the broker client instead") + return nil, errors.New("GetACLs not yet supported with zk access mode; omit zk addresses to fix.") } // UpdateTopicConfig updates the config JSON for a topic and sets a change From 90147f39dc7507d51097119c8be088e595ec3df7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:40:56 -0400 Subject: [PATCH 036/116] clean up createacl --- pkg/admin/client.go | 6 ++++++ pkg/admin/zkclient.go | 9 ++++++++- pkg/admin/zkclient_test.go | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/admin/client.go b/pkg/admin/client.go index c3b4bf8c..5ec1f731 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -68,6 +68,12 @@ type Client interface { config kafka.TopicConfig, ) error + // Create ACL creates an ACL in the cluster. + CreateACL( + ctx context.Context, + entry kafka.ACLEntry, + ) error + // AssignPartitions sets the replica broker IDs for one or more partitions in a topic. AssignPartitions( ctx context.Context, diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 5a6f115e..0aca8e77 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -426,7 +426,14 @@ func (c *ZKAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, ) ([]ACLInfo, error) { - return nil, errors.New("GetACLs not yet supported with zk access mode; omit zk addresses to fix.") + return nil, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") +} + +func (c *ZKAdminClient) CreateACL( + ctx context.Context, + entry kafka.ACLEntry, +) error { + return errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } // UpdateTopicConfig updates the config JSON for a topic and sets a change diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index d2adf21d..f0a692c4 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1091,3 +1091,19 @@ func TestZkGetACLs(t *testing.T) { assert.Empty(t, acls) assert.Error(t, err) } + +func TestZkCreateACL(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + BootstrapAddrs: []string{util.TestKafkaAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) + assert.Error(t, err) +} From 7534ecfc900bf7d308bbf2ce132b6cc8903538fe Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:47:29 -0400 Subject: [PATCH 037/116] add TestBrokerClientCreateACLReadOnly --- pkg/admin/brokerclient_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 66f25ab6..ea159fa9 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -2,6 +2,7 @@ package admin import ( "context" + "errors" "fmt" "testing" "time" @@ -652,3 +653,21 @@ func TestBrokerClientCreateGetACL(t *testing.T) { } assert.Equal(t, expected, aclsInfo) } + +func TestBrokerClientCreateACLReadOnly(t *testing.T) { + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ReadOnly: true, + }, + ) + require.NoError(t, err) + + err = client.CreateACL(ctx, kafka.ACLEntry{}) + assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) + +} From 7551ece928980a497a8112756980d69e2554b0c1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:50:52 -0400 Subject: [PATCH 038/116] improve zk tests --- pkg/admin/zkclient_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index f0a692c4..82a873c7 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -2,6 +2,7 @@ package admin import ( "context" + "errors" "fmt" "testing" "time" @@ -1089,7 +1090,7 @@ func TestZkGetACLs(t *testing.T) { acls, err := adminClient.GetACLs(ctx, kafka.ACLFilter{}) assert.Empty(t, acls) - assert.Error(t, err) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } func TestZkCreateACL(t *testing.T) { @@ -1105,5 +1106,5 @@ func TestZkCreateACL(t *testing.T) { defer adminClient.Close() err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) - assert.Error(t, err) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } From df19f187bb242199741398356bb61a4ec1b2f554 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:14:21 -0400 Subject: [PATCH 039/116] run acl tests in ci --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44329a7b..5cd5b31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,7 @@ jobs: env: KAFKA_TOPICS_TEST_ZK_ADDR: zookeeper:2181 KAFKA_TOPICS_TEST_KAFKA_ADDR: kafka1:9092 + KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY: 1 services: zookeeper: From e799c4019d33d7ceed6746d99d977ecbff68ee7f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:16:33 -0400 Subject: [PATCH 040/116] enable acls for kafka 2.4.1 in ci --- .github/workflows/ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cd5b31a..ac9d46ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka1 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka2: image: wurstmeister/kafka:2.12-2.4.1 @@ -168,6 +170,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka2 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka3: image: wurstmeister/kafka:2.12-2.4.1 @@ -179,6 +183,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka3 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka4: image: wurstmeister/kafka:2.12-2.4.1 @@ -190,6 +196,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka4 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka5: image: wurstmeister/kafka:2.12-2.4.1 @@ -201,6 +209,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka5 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka6: image: wurstmeister/kafka:2.12-2.4.1 @@ -212,7 +222,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka6 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true snyk: runs-on: ubuntu-latest From cf690ee6bc9f0ac670ebddb4a1fe48c4383094f5 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:26:48 -0400 Subject: [PATCH 041/116] fix zk tests --- pkg/admin/zkclient_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 82a873c7..34383b7d 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1081,8 +1081,7 @@ func TestZkGetACLs(t *testing.T) { adminClient, err := NewZKAdminClient( ctx, ZKAdminClientConfig{ - ZKAddrs: []string{util.TestZKAddr()}, - BootstrapAddrs: []string{util.TestKafkaAddr()}, + ZKAddrs: []string{util.TestZKAddr()}, }, ) require.NoError(t, err) @@ -1098,8 +1097,7 @@ func TestZkCreateACL(t *testing.T) { adminClient, err := NewZKAdminClient( ctx, ZKAdminClientConfig{ - ZKAddrs: []string{util.TestZKAddr()}, - BootstrapAddrs: []string{util.TestKafkaAddr()}, + ZKAddrs: []string{util.TestZKAddr()}, }, ) require.NoError(t, err) From 41283c7b7f49465ed3df8b41ab3aa74052aa7627 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:35:19 -0400 Subject: [PATCH 042/116] skip TestBrokerClientCreateACLReadOnly on old versions of kafka --- pkg/admin/brokerclient_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index ea159fa9..7f17ced3 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -655,6 +655,9 @@ func TestBrokerClientCreateGetACL(t *testing.T) { } func TestBrokerClientCreateACLReadOnly(t *testing.T) { + if !util.CanTestBrokerAdmin() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } ctx := context.Background() client, err := NewBrokerAdminClient( ctx, From b553d3d9f44d2e896e23b54ad3465774ab483dd2 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:38:40 -0400 Subject: [PATCH 043/116] try to debug --- pkg/admin/brokerclient.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 14b73e26..8a593650 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -745,6 +745,8 @@ func (c *BrokerAdminClient) CreateACL( return errors.New("Cannot create ACL in read-only mode") } + log.SetLevel(log.DebugLevel) + req := kafka.CreateACLsRequest{ ACLs: []kafka.ACLEntry{ entry, From 14811f7ebbf07c3d13c260bce57c7ddca4404b3e Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:51:11 -0400 Subject: [PATCH 044/116] handle nested errors from createacls --- pkg/admin/brokerclient.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 8a593650..b6df16c9 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -759,8 +759,14 @@ func (c *BrokerAdminClient) CreateACL( if err != nil { return err } - if len(resp.Errors) > 0 { - return fmt.Errorf("%+v", resp.Errors) + var errors []error + for _, err := range resp.Errors { + if err != nil { + errors = append(errors, err) + } + } + if len(errors) > 0 { + return fmt.Errorf("%+v", errors) } return nil } From 7cc16a652a7a8a047cce5fa72843931e527c6ca1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 16:08:10 -0400 Subject: [PATCH 045/116] operations -> operation --- cmd/topicctl/subcmd/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 5461397a..ba3b946e 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -303,7 +303,7 @@ $ topicctl get acls --host 198.51.100.0 ) cmd.Flags().Var( &aclsConfig.operationType, - "operations", + "operation", `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, ) cmd.Flags().Var( From 2d126423a28ff30e5541b735972d3d0d30801cc1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 16:08:25 -0400 Subject: [PATCH 046/116] operations -> operation --- cmd/topicctl/subcmd/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index ba3b946e..068d35b6 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -269,7 +269,7 @@ func aclsCmd() *cobra.Command { $ topicctl get acls List read acls for topic my-topic -$ topicctl get acls --resource-type topic --resource-name my-topic --operations read +$ topicctl get acls --resource-type topic --resource-name my-topic --operation read List acls for user Alice with permission allow $ topicctl get acls --principal User:alice --permission-type allow From fdb8288a2f4c502835242e9de628540e7101a76d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 16:23:16 -0400 Subject: [PATCH 047/116] remove setting log level in test --- pkg/admin/brokerclient.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index b6df16c9..5211e87b 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -745,8 +745,6 @@ func (c *BrokerAdminClient) CreateACL( return errors.New("Cannot create ACL in read-only mode") } - log.SetLevel(log.DebugLevel) - req := kafka.CreateACLsRequest{ ACLs: []kafka.ACLEntry{ entry, From 96dedfd7c4322789bfba93722ca8a907715623c0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 18 Sep 2023 14:51:43 -0400 Subject: [PATCH 048/116] clean up allowed types in help command --- cmd/topicctl/subcmd/get.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index e3f0390d..53e369fd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -342,12 +342,12 @@ $ topicctl get acls --host 198.51.100.0 cmd.Flags().Var( &aclsConfig.operationType, "operation", - `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, + `The operation that is being allowed or denied to filter on. allowed: [any, all, read, write, create, delete, alter, describe, clusteraction, describeconfigs, alterconfigs, idempotentwrite]`, ) cmd.Flags().Var( &aclsConfig.permissionType, "permission-type", - `The permission type to filter on. allowed: "any", "allow", or "deny"`, + `The permission type to filter on. allowed: [any, allow, deny]`, ) cmd.Flags().StringVar( &aclsConfig.principalFilter, @@ -364,12 +364,12 @@ $ topicctl get acls --host 198.51.100.0 cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", - `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed". "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, + `The type of the resource pattern or filter. allowed: [any, match, literal, prefixed]. "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, ) cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", - `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + `The type of resource to filter on. allowed: [any, topic, group, cluster, transactionalid, delegationtoken]`, ) return cmd } From d65759d2b38323fd7fdc6b393a3578499b6a1311 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 18 Sep 2023 14:57:34 -0400 Subject: [PATCH 049/116] fix merge conflict --- cmd/topicctl/subcmd/get.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 53e369fd..4c00e7d8 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -316,10 +316,16 @@ List acls for host 198.51.100.0 $ topicctl get acls --host 198.51.100.0 `, RunE: func(cmd *cobra.Command, args []string) error { - ctx, cliRunner, err := getCliRunnerAndCtx() + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) if err != nil { return err } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) filter := kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), From 36d3de99ca10825a476d9abfbe551919ab4f53fc Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 13:36:42 -0400 Subject: [PATCH 050/116] fix test --- pkg/admin/brokerclient_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 7f17ced3..e88eea25 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -656,7 +656,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { func TestBrokerClientCreateACLReadOnly(t *testing.T) { if !util.CanTestBrokerAdmin() { - t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") } ctx := context.Background() client, err := NewBrokerAdminClient( From 9b1262a71ebdd9e5f3d00af8a7bce2801b3a5208 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 15:24:27 -0400 Subject: [PATCH 051/116] add json annotations --- pkg/admin/types.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 14b7c744..945c0999 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -78,13 +78,13 @@ type PartitionAssignment struct { // PartitionInfo represents the information stored about an ACL // in zookeeper. type ACLInfo struct { - ResourceType ResourceType - ResourceName string - PatternType PatternType - Principal string - Host string - Operation ACLOperationType - PermissionType ACLPermissionType + ResourceType ResourceType `json:"resourceType"` + ResourceName string `json:"resourceName"` + PatternType PatternType `json:"patternType"` + Principal string `json:"principal"` + Host string `json:"host"` + Operation ACLOperationType `json:"operation"` + PermissionType ACLPermissionType `json:"permissionType"` } // ResourceType presents the Kafka resource type. From e960c370bc96a472394c7378bbec769ed9a031b0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 18 Sep 2023 15:00:32 -0400 Subject: [PATCH 052/116] bump kafka-go to version on main --- go.sum | 8 -------- 1 file changed, 8 deletions(-) diff --git a/go.sum b/go.sum index 6e87f76d..2e840f8a 100644 --- a/go.sum +++ b/go.sum @@ -32,7 +32,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -73,9 +72,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= -github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= -github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 h1:Gok6q1P5yUVLYt+auyZgcRBP89thqCmU9MNT65Ms1SI= @@ -118,7 +114,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,12 +135,10 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -153,7 +146,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 47650f95e86b0e53c82cff842949325e67f780e7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 13:36:08 -0400 Subject: [PATCH 053/116] wip --- pkg/admin/brokerclient.go | 59 ++++++++++++++++++++++++++++++++++ pkg/admin/brokerclient_test.go | 26 ++++++++++++++- pkg/admin/client.go | 12 +++++++ pkg/admin/types.go | 6 ++++ pkg/admin/zkclient.go | 14 ++++++++ pkg/util/error.go | 15 +++++++++ 6 files changed, 131 insertions(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 5211e87b..2d697fe6 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -379,6 +379,65 @@ func (c *BrokerAdminClient) GetTopic( return topicInfos[0], nil } +func (c *BrokerAdminClient) GetUsers( + ctx context.Context, + names []string, +) ([]UserInfo, error) { + var users []kafka.UserScramCredentialsUser + for _, name := range names { + users = append(users, kafka.UserScramCredentialsUser{ + Name: name, + }) + } + + req := kafka.DescribeUserScramCredentialsRequest{ + Users: users, + } + log.Debugf("DescribeUserScramCredentials request: %+v", req) + + resp, err := c.client.DescribeUserScramCredentials(ctx, &req) + log.Debugf("DescribeUserScramCredentials response: %+v (%+v)", resp, err) + if err != nil { + return nil, err + } + + if err = util.DescribeUserScramCredentialsResponseResultsError(resp.Results); err != nil { + return nil, err + } + + results := []UserInfo{} + + for _, result := range resp.Results { + results = append(results, UserInfo{ + Name: result.User, + CredentialInfos: result.CredentialInfos, + }) + } + return results, err +} + +func (c *BrokerAdminClient) CreateUser( + ctx context.Context, + user kafka.UserScramCredentialsUpsertion, +) error { + if c.config.ReadOnly { + return errors.New("Cannot create user in read-only mode") + } + req := kafka.AlterUserScramCredentialsRequest{ + Upsertions: []kafka.UserScramCredentialsUpsertion{user}, + } + log.Debugf("AlterUserScramCredentials request: %+v", req) + resp, err := c.client.AlterUserScramCredentials(ctx, &req) + log.Debugf("AlterUserScramCredentials response: %+v", resp) + if err != nil { + return err + } + if err = resp.Results[0].Error; err != nil { + return err + } + return nil +} + // UpdateTopicConfig updates the configuration for the argument topic. It returns the config // keys that were updated. func (c *BrokerAdminClient) UpdateTopicConfig( diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index e88eea25..6d4c4526 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -670,7 +670,31 @@ func TestBrokerClientCreateACLReadOnly(t *testing.T) { ) require.NoError(t, err) + err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{}) + + assert.Equal(t, err, errors.New("Cannot create user in read-only mode.")) +} + +func TestBrokerClientCreateGetUsers(t *testing.T) { + +} + +func TestBrokerClientCreateUserReadOnly(t *testing.T) { + if !util.CanTestBrokerAdmin() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") + } + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ReadOnly: true, + }, + ) + require.NoError(t, err) + err = client.CreateACL(ctx, kafka.ACLEntry{}) assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) - } diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 5ec1f731..d94bb46b 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -44,6 +44,12 @@ type Client interface { filter kafka.ACLFilter, ) ([]ACLInfo, error) + // GetUsers gets information about users in the cluster. + GetUsers( + ctx context.Context, + names []string, + ) ([]UserInfo, error) + // UpdateTopicConfig updates the configuration for the argument topic. It returns the config // keys that were updated. UpdateTopicConfig( @@ -74,6 +80,12 @@ type Client interface { entry kafka.ACLEntry, ) error + // CreateUser creates a user in zookeeper. + CreateUser( + ctx context.Context, + user kafka.UserScramCredentialsUpsertion, + ) error + // AssignPartitions sets the replica broker IDs for one or more partitions in a topic. AssignPartitions( ctx context.Context, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 945c0999..4e117772 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -288,6 +288,12 @@ func (p *ACLPermissionType) Set(v string) error { // Type is used by Cobra in help text. func (p *ACLPermissionType) Type() string { return "ACLPermissionType" + +// UserInfo represents the information stored about a user +// in zookeeper. +type UserInfo struct { + Name string + CredentialInfos []kafka.DescribeUserScramCredentialsCredentialInfo } type zkClusterID struct { diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 0aca8e77..9a93b997 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -436,6 +436,20 @@ func (c *ZKAdminClient) CreateACL( return errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } +func (c *ZKAdminClient) GetUsers( + ctx context.Context, + names []string, +) ([]UserInfo, error) { + return nil, errors.New("Users not yet supported with zk access mode; omit zk addresses to fix.") +} + +func (c *ZKAdminClient) CreateUser( + ctx context.Context, + user kafka.UserScramCredentialsUpsertion, +) error { + return errors.New("Users not yet supported with zk access mode; omit zk addresses to fix.") +} + // UpdateTopicConfig updates the config JSON for a topic and sets a change // notification so that the brokers are notified. If overwrite is true, then // it will overwrite existing config entries. diff --git a/pkg/util/error.go b/pkg/util/error.go index 41cdfb6c..d721fffb 100644 --- a/pkg/util/error.go +++ b/pkg/util/error.go @@ -49,3 +49,18 @@ func AlterPartitionReassignmentsRequestAssignmentError(results []kafka.AlterPart } return nil } + +func DescribeUserScramCredentialsResponseResultsError(results []kafka.DescribeUserScramCredentialsResponseResult) error { + errors := map[string]error{} + var hasErrors bool + for _, result := range results { + if result.Error != nil { + hasErrors = true + errors[result.User] = result.Error + } + } + if hasErrors { + return fmt.Errorf("%+v", errors) + } + return nil +} From 8d9ab94e5690f9c4a11f19d7da5474edbff49915 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 13:51:04 -0400 Subject: [PATCH 054/116] basic tests --- docker-compose-auth.yml | 2 +- pkg/admin/brokerclient_test.go | 130 +++++++++++++++++++++++++++++++-- pkg/admin/zkclient_test.go | 31 ++++++++ 3 files changed, 154 insertions(+), 9 deletions(-) diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index 7897cb8d..22d46a46 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -16,7 +16,7 @@ services: - "2181:2181" kafka: - image: wurstmeister/kafka:2.12-2.4.1 + image: wurstmeister/kafka:2.13-2.7.1 restart: on-failure:3 links: - zookeeper diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 6d4c4526..1e437169 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -654,9 +654,9 @@ func TestBrokerClientCreateGetACL(t *testing.T) { assert.Equal(t, expected, aclsInfo) } -func TestBrokerClientCreateACLReadOnly(t *testing.T) { - if !util.CanTestBrokerAdmin() { - t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") +func TestBrokerClientCreateGetUsers(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") } ctx := context.Background() client, err := NewBrokerAdminClient( @@ -665,18 +665,100 @@ func TestBrokerClientCreateACLReadOnly(t *testing.T) { ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, - ReadOnly: true, }, ) require.NoError(t, err) - err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{}) + err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{ + Name: "junk", + Mechanism: kafka.ScramMechanismSha512, + Iterations: 15000, + Salt: []byte("my-salt"), + SaltedPassword: []byte("my-salted-password"), + }) + + require.NoError(t, err) - assert.Equal(t, err, errors.New("Cannot create user in read-only mode.")) + resp, err := client.GetUsers(ctx, []string{"junk"}) + require.NoError(t, err) + assert.Equal(t, []UserInfo{ + { + Name: "junk", + CredentialInfos: []kafka.DescribeUserScramCredentialsCredentialInfo{ + { + Mechanism: kafka.ScramMechanismSha512, + Iterations: 15000, + }, + }, + }, + }, resp) } func TestBrokerClientCreateGetUsers(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, + ) + require.NoError(t, err) + + name := util.RandomString("test-user-", 6) + mechanism := kafka.ScramMechanismSha512 + + defer func() { + resp, err := client.client.AlterUserScramCredentials( + ctx, + &kafka.AlterUserScramCredentialsRequest{ + Deletions: []kafka.UserScramCredentialsDeletion{ + { + Name: name, + Mechanism: mechanism, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up user, err: %v", err)) + } + for _, response := range resp.Results { + if err = response.Error; err != nil { + t.Fatal(fmt.Errorf("failed to clean up user, err: %v", err)) + } + } + + }() + + err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{ + Name: name, + Mechanism: mechanism, + Iterations: 15000, + Salt: []byte("my-salt"), + SaltedPassword: []byte("my-salted-password"), + }) + + require.NoError(t, err) + resp, err := client.GetUsers(ctx, []string{name}) + require.NoError(t, err) + assert.Equal(t, []UserInfo{ + { + Name: name, + CredentialInfos: []CredentialInfo{ + { + ScramMechanism: ScramMechanism(mechanism), + Iterations: 15000, + }, + }, + }, + }, resp) } func TestBrokerClientCreateUserReadOnly(t *testing.T) { @@ -695,6 +777,38 @@ func TestBrokerClientCreateUserReadOnly(t *testing.T) { ) require.NoError(t, err) - err = client.CreateACL(ctx, kafka.ACLEntry{}) - assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) + err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{}) + + assert.Equal(t, errors.New("Cannot create user in read-only mode"), err) +} + +func TestZkGetACLs(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + acls, err := adminClient.GetACLs(ctx, kafka.ACLFilter{}) + assert.Empty(t, acls) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) +} + +func TestZkCreateACL(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 34383b7d..8c87e1ab 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1106,3 +1106,34 @@ func TestZkCreateACL(t *testing.T) { err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } + +func TestZkGetUsers(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + acls, err := adminClient.GetUsers(ctx, []string{}) + assert.Empty(t, acls) + assert.Equal(t, err, errors.New("Users not yet supported with zk access mode; omit zk addresses to fix.")) +} + +func TestZkCreateUser(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + err = adminClient.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{}) + assert.Equal(t, err, errors.New("Users not yet supported with zk access mode; omit zk addresses to fix.")) +} From 561eb2a147d74bc0c416470e4130296aad2a92e2 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 15:23:09 -0400 Subject: [PATCH 055/116] start on getusers cmd --- cmd/topicctl/subcmd/get.go | 16 ++++++++++++++ pkg/admin/format.go | 45 ++++++++++++++++++++++++++++++++++++++ pkg/cli/cli.go | 15 +++++++++++++ 3 files changed, 76 insertions(+) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 4c00e7d8..a6d173f1 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -58,6 +58,7 @@ func init() { offsetsCmd(), topicsCmd(), aclsCmd(), + usersCmd(), ) RootCmd.AddCommand(getCmd) } @@ -379,3 +380,18 @@ $ topicctl get acls --host 198.51.100.0 ) return cmd } + +func usersCmd() *cobra.Command { + return &cobra.Command{ + Use: "users", + Short: "Displays information for all users in the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cliRunner, err := getCliRunnerAndCtx() + if err != nil { + return err + } + return cliRunner.GetUsers(ctx, getConfig.full) + }, + } +} diff --git a/pkg/admin/format.go b/pkg/admin/format.go index fe0625d0..b80716fb 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -801,6 +801,51 @@ func FormatACLs(acls []ACLInfo) string { return string(bytes.TrimRight(buf.Bytes(), "\n")) } +// FormatUsers creates a pretty table that lists the details of the +// argument users. +func FormatUsers(users []UserInfo) string { + buf := &bytes.Buffer{} + + headers := []string{ + "Name", + "Mechanism", + "Iterations", + } + + table := tablewriter.NewWriter(buf) + table.SetHeader(headers) + table.SetAutoWrapText(false) + table.SetColumnAlignment( + []int{ + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + }, + ) + table.SetBorders( + tablewriter.Border{ + Left: false, + Top: true, + Right: false, + Bottom: true, + }, + ) + + for _, user := range users { + for _, credential := range user.CredentialInfos { + row := []string{ + user.Name, + credential.ScramMechanism.String(), + fmt.Sprintf("%d", credential.Iterations), + } + + table.Append(row) + } + } + + table.Render() + return string(bytes.TrimRight(buf.Bytes(), "\n")) +} + func prettyConfig(config map[string]string) string { rows := []string{} diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 84b37bd6..6dd1f685 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -488,6 +488,21 @@ func (c *CLIRunner) GetTopics(ctx context.Context, full bool) error { return nil } +// GerUsers fetches the details of each user in the cluster and prints out a table of them. +func (c *CLIRunner) GetUsers(ctx context.Context, names []string) error { + c.startSpinner() + + users, err := c.adminClient.GetUsers(ctx, names) + c.stopSpinner() + if err != nil { + return err + } + + c.printer("Users:\n%s", admin.FormatUsers(users)) + + return nil +} + // ResetOffsets resets the offsets for a single consumer group / topic combination. func (c *CLIRunner) ResetOffsets( ctx context.Context, From ead5d316aeb11c122dd45aa4d67f073574a7e14a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 15:25:01 -0400 Subject: [PATCH 056/116] add json annotations --- pkg/admin/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 4e117772..749dfe38 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -292,8 +292,8 @@ func (p *ACLPermissionType) Type() string { // UserInfo represents the information stored about a user // in zookeeper. type UserInfo struct { - Name string - CredentialInfos []kafka.DescribeUserScramCredentialsCredentialInfo + Name string `json:"name"` + CredentialInfos []kafka.DescribeUserScramCredentialsCredentialInfo `json:"credentialInfos"` } type zkClusterID struct { From 5dcb773ebf8c05595402ff671b804f14f0172d11 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 15:47:02 -0400 Subject: [PATCH 057/116] get users working --- cmd/topicctl/subcmd/get.go | 2 +- pkg/admin/brokerclient.go | 9 ++++++++- pkg/admin/brokerclient_test.go | 6 +++--- pkg/admin/types.go | 26 ++++++++++++++++++++++++-- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index a6d173f1..f1be9410 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -391,7 +391,7 @@ func usersCmd() *cobra.Command { if err != nil { return err } - return cliRunner.GetUsers(ctx, getConfig.full) + return cliRunner.GetUsers(ctx, nil) }, } } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 2d697fe6..a31a8e52 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -408,9 +408,16 @@ func (c *BrokerAdminClient) GetUsers( results := []UserInfo{} for _, result := range resp.Results { + var credentials []CredentialInfo + for _, credential := range result.CredentialInfos { + credentials = append(credentials, CredentialInfo{ + ScramMechanism: ScramMechanism(credential.Mechanism), + Iterations: credential.Iterations, + }) + } results = append(results, UserInfo{ Name: result.User, - CredentialInfos: result.CredentialInfos, + CredentialInfos: credentials, }) } return results, err diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 1e437169..a53d499b 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -684,10 +684,10 @@ func TestBrokerClientCreateGetUsers(t *testing.T) { assert.Equal(t, []UserInfo{ { Name: "junk", - CredentialInfos: []kafka.DescribeUserScramCredentialsCredentialInfo{ + CredentialInfos: []CredentialInfo{ { - Mechanism: kafka.ScramMechanismSha512, - Iterations: 15000, + ScramMechanism: ScramMechanism(kafka.ScramMechanismSha512), + Iterations: 15000, }, }, }, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 749dfe38..3566f758 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -292,8 +292,30 @@ func (p *ACLPermissionType) Type() string { // UserInfo represents the information stored about a user // in zookeeper. type UserInfo struct { - Name string `json:"name"` - CredentialInfos []kafka.DescribeUserScramCredentialsCredentialInfo `json:"credentialInfos"` + Name string `json:"name"` + CredentialInfos []CredentialInfo `json:"credentialInfos"` +} + +// CredentialInfo represents read only information about +// a users credentials in zookeeper. +type CredentialInfo struct { + ScramMechanism ScramMechanism + Iterations int +} + +// ScramMechanism represents the ScramMechanism used +// for a users credential in zookeeper. +type ScramMechanism kafka.ScramMechanism + +func (s *ScramMechanism) String() string { + switch kafka.ScramMechanism(*s) { + case kafka.ScramMechanismSha256: + return "sha256" + case kafka.ScramMechanismSha512: + return "sha512" + default: + return "unknown" + } } type zkClusterID struct { From 46a50ef8cf4956cde9b7498cde07879320d94042 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 15:53:10 -0400 Subject: [PATCH 058/116] wip --- cmd/topicctl/subcmd/get.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index f1be9410..3f71f725 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -387,10 +387,16 @@ func usersCmd() *cobra.Command { Short: "Displays information for all users in the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx, cliRunner, err := getCliRunnerAndCtx() + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) if err != nil { return err } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetUsers(ctx, nil) }, } From 680011437518d29b99dc4fd881dbacb2bb3f981e Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 16:19:07 -0400 Subject: [PATCH 059/116] add todos and fix type annotaitons --- pkg/admin/brokerclient_test.go | 3 +++ pkg/admin/types.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index a53d499b..232826fc 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -679,6 +679,9 @@ func TestBrokerClientCreateGetUsers(t *testing.T) { require.NoError(t, err) + // TODO: use randomly generated name for user + // TODO: delete user + resp, err := client.GetUsers(ctx, []string{"junk"}) require.NoError(t, err) assert.Equal(t, []UserInfo{ diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 3566f758..6595aa3f 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -299,8 +299,8 @@ type UserInfo struct { // CredentialInfo represents read only information about // a users credentials in zookeeper. type CredentialInfo struct { - ScramMechanism ScramMechanism - Iterations int + ScramMechanism ScramMechanism `json:"scramMechanism"` + Iterations int `json:"iterations"` } // ScramMechanism represents the ScramMechanism used From 2b0d87cda4330385b86b3908b03556d9188536f4 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 16:25:53 -0400 Subject: [PATCH 060/116] improve test --- pkg/admin/brokerclient_test.go | 40 +++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 232826fc..e292529a 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -669,9 +669,36 @@ func TestBrokerClientCreateGetUsers(t *testing.T) { ) require.NoError(t, err) + name := util.RandomString("test-user-", 6) + mechanism := kafka.ScramMechanismSha512 + + defer func() { + resp, err := client.client.AlterUserScramCredentials( + ctx, + &kafka.AlterUserScramCredentialsRequest{ + Deletions: []kafka.UserScramCredentialsDeletion{ + { + Name: name, + Mechanism: mechanism, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up user, err: %v", err)) + } + for _, response := range resp.Results { + if err = response.Error; err != nil { + t.Fatal(fmt.Errorf("failed to clean up user, err: %v", err)) + } + } + + }() + err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{ - Name: "junk", - Mechanism: kafka.ScramMechanismSha512, + Name: name, + Mechanism: mechanism, Iterations: 15000, Salt: []byte("my-salt"), SaltedPassword: []byte("my-salted-password"), @@ -679,17 +706,14 @@ func TestBrokerClientCreateGetUsers(t *testing.T) { require.NoError(t, err) - // TODO: use randomly generated name for user - // TODO: delete user - - resp, err := client.GetUsers(ctx, []string{"junk"}) + resp, err := client.GetUsers(ctx, []string{name}) require.NoError(t, err) assert.Equal(t, []UserInfo{ { - Name: "junk", + Name: name, CredentialInfos: []CredentialInfo{ { - ScramMechanism: ScramMechanism(kafka.ScramMechanismSha512), + ScramMechanism: ScramMechanism(mechanism), Iterations: 15000, }, }, From 128be0dd8854d66433eac861f9969b1de064586f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 16:31:15 -0400 Subject: [PATCH 061/116] use CanTestBrokerAdminSecurity to feature flag test --- pkg/admin/brokerclient_test.go | 67 ---------------------------------- 1 file changed, 67 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index e292529a..f72de5cf 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -721,73 +721,6 @@ func TestBrokerClientCreateGetUsers(t *testing.T) { }, resp) } -func TestBrokerClientCreateGetUsers(t *testing.T) { - if !util.CanTestBrokerAdminSecurity() { - t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") - } - ctx := context.Background() - client, err := NewBrokerAdminClient( - ctx, - BrokerAdminClientConfig{ - ConnectorConfig: ConnectorConfig{ - BrokerAddr: util.TestKafkaAddr(), - }, - }, - ) - require.NoError(t, err) - - name := util.RandomString("test-user-", 6) - mechanism := kafka.ScramMechanismSha512 - - defer func() { - resp, err := client.client.AlterUserScramCredentials( - ctx, - &kafka.AlterUserScramCredentialsRequest{ - Deletions: []kafka.UserScramCredentialsDeletion{ - { - Name: name, - Mechanism: mechanism, - }, - }, - }, - ) - - if err != nil { - t.Fatal(fmt.Errorf("failed to clean up user, err: %v", err)) - } - for _, response := range resp.Results { - if err = response.Error; err != nil { - t.Fatal(fmt.Errorf("failed to clean up user, err: %v", err)) - } - } - - }() - - err = client.CreateUser(ctx, kafka.UserScramCredentialsUpsertion{ - Name: name, - Mechanism: mechanism, - Iterations: 15000, - Salt: []byte("my-salt"), - SaltedPassword: []byte("my-salted-password"), - }) - - require.NoError(t, err) - - resp, err := client.GetUsers(ctx, []string{name}) - require.NoError(t, err) - assert.Equal(t, []UserInfo{ - { - Name: name, - CredentialInfos: []CredentialInfo{ - { - ScramMechanism: ScramMechanism(mechanism), - Iterations: 15000, - }, - }, - }, - }, resp) -} - func TestBrokerClientCreateUserReadOnly(t *testing.T) { if !util.CanTestBrokerAdmin() { t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") From a69e71f00b822843504569f2985fa8b0b0b78cc0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 16:44:33 -0400 Subject: [PATCH 062/116] update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5701dbff..3657f32f 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ resource type in the cluster. Currently, the following operations are supported: | `get offsets [topic]` | Number of messages per partition along with start and end times | | `get topics` | All topics in the cluster | | `get acls [flags]` | Describe access control levels (ACLs) in the cluster | +| `get users` | All users in the cluster | #### rebalance From 5efedaaca5475724177b37bca046dfbc64d59314 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 20 Sep 2023 09:50:40 -0400 Subject: [PATCH 063/116] remove duplicate test from merge conflicts --- pkg/admin/brokerclient_test.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index f72de5cf..5d6103a4 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -741,34 +741,3 @@ func TestBrokerClientCreateUserReadOnly(t *testing.T) { assert.Equal(t, errors.New("Cannot create user in read-only mode"), err) } - -func TestZkGetACLs(t *testing.T) { - ctx := context.Background() - adminClient, err := NewZKAdminClient( - ctx, - ZKAdminClientConfig{ - ZKAddrs: []string{util.TestZKAddr()}, - }, - ) - require.NoError(t, err) - defer adminClient.Close() - - acls, err := adminClient.GetACLs(ctx, kafka.ACLFilter{}) - assert.Empty(t, acls) - assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) -} - -func TestZkCreateACL(t *testing.T) { - ctx := context.Background() - adminClient, err := NewZKAdminClient( - ctx, - ZKAdminClientConfig{ - ZKAddrs: []string{util.TestZKAddr()}, - }, - ) - require.NoError(t, err) - defer adminClient.Close() - - err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) - assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) -} From 83eca681ba02354c8aeb3e17ed50a44434f1880f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 20 Sep 2023 09:57:03 -0400 Subject: [PATCH 064/116] fix more merge conflicts --- go.sum | 8 ++++++-- pkg/admin/types.go | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index 2e840f8a..2a92dd13 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -72,8 +73,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= -github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= +github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 h1:Gok6q1P5yUVLYt+auyZgcRBP89thqCmU9MNT65Ms1SI= github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= @@ -114,6 +114,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -135,10 +136,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -146,6 +149,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 6595aa3f..73861a06 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -288,7 +288,8 @@ func (p *ACLPermissionType) Set(v string) error { // Type is used by Cobra in help text. func (p *ACLPermissionType) Type() string { return "ACLPermissionType" - +} + // UserInfo represents the information stored about a user // in zookeeper. type UserInfo struct { From a5dbb567cb83ddc76b110d0b8913fc8e33aacbc5 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 26 Sep 2023 10:37:26 -0400 Subject: [PATCH 065/116] create user working --- .../local-cluster/topics/topic-default.yaml | 2 +- pkg/admin/brokerclient.go | 14 ++-- pkg/cli/cli.go | 32 +++++++++ pkg/config/load.go | 40 +++++++++++ pkg/config/load_test.go | 68 +++++++++++++++++++ pkg/util/error.go | 15 ---- 6 files changed, 150 insertions(+), 21 deletions(-) diff --git a/examples/local-cluster/topics/topic-default.yaml b/examples/local-cluster/topics/topic-default.yaml index a7024142..97b5b223 100644 --- a/examples/local-cluster/topics/topic-default.yaml +++ b/examples/local-cluster/topics/topic-default.yaml @@ -11,7 +11,7 @@ spec: replicationFactor: 2 retentionMinutes: 100 placement: - strategy: in-rack + strategy: any settings: cleanup.policy: delete max.message.bytes: 5542880 diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index a31a8e52..e616fc1d 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -401,13 +401,17 @@ func (c *BrokerAdminClient) GetUsers( return nil, err } - if err = util.DescribeUserScramCredentialsResponseResultsError(resp.Results); err != nil { - return nil, err - } - results := []UserInfo{} for _, result := range resp.Results { + if result.Error != nil { + log.Debugf("got here") + if errors.Is(result.Error, kafka.ResourceNotFound) { + log.Debugf("Skipping over user %s because it does not exist", result.User) + continue + } + return nil, fmt.Errorf("Error getting description of user %s: %+v", result.User, result.Error) + } var credentials []CredentialInfo for _, credential := range result.CredentialInfos { credentials = append(credentials, CredentialInfo{ @@ -420,7 +424,7 @@ func (c *BrokerAdminClient) GetUsers( CredentialInfos: credentials, }) } - return results, err + return results, nil } func (c *BrokerAdminClient) CreateUser( diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 6dd1f685..5b920ec2 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -111,6 +111,38 @@ func (c *CLIRunner) ApplyTopic( return nil } +// ApplyUser does an apply run according to the spec in the argument config. +func (c *CLIRunner) ApplyUser( + ctx context.Context, + applierConfig apply.UserApplierConfig, +) error { + applier, err := apply.NewUserApplier( + ctx, + c.adminClient, + applierConfig, + ) + if err != nil { + return err + } + + highlighter := color.New(color.FgYellow, color.Bold).SprintfFunc() + + c.printer( + "Starting apply for user %s in environment %s, cluster %s", + highlighter(applierConfig.UserConfig.Meta.Name), + highlighter(applierConfig.UserConfig.Meta.Environment), + highlighter(applierConfig.UserConfig.Meta.Cluster), + ) + + err = applier.Apply(ctx) + if err != nil { + return err + } + + c.printer("Apply completed successfully!") + return nil +} + // BootstrapTopics creates configs for one or more topics based on their current state in the // cluster. func (c *CLIRunner) BootstrapTopics( diff --git a/pkg/config/load.go b/pkg/config/load.go index 2e7848b1..3ec65ae1 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io/ioutil" "os" "path/filepath" @@ -83,6 +84,45 @@ func LoadTopicsFile(path string) ([]TopicConfig, error) { func LoadTopicBytes(contents []byte) (TopicConfig, error) { config := TopicConfig{} err := unmarshalYAMLStrict(contents, &config) + fmt.Println(config) + return config, err +} + +// LoadUsersFile loads one or more UserConfigs from a path to a YAML file. +func LoadUsersFile(path string) ([]UserConfig, error) { + contents, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + contents = []byte(os.ExpandEnv(string(contents))) + + trimmedFile := strings.TrimSpace(string(contents)) + userStrs := sep.Split(trimmedFile, -1) + + userConfigs := []UserConfig{} + + for _, userStr := range userStrs { + userStr = strings.TrimSpace(userStr) + if isEmpty(userStr) { + continue + } + + userConfig, err := LoadUserBytes([]byte(userStr)) + if err != nil { + return nil, err + } + + userConfigs = append(userConfigs, userConfig) + } + + return userConfigs, nil +} + +// LoadUserBytes loads a UserConfig from YAML bytes. +func LoadUserBytes(contents []byte) (UserConfig, error) { + config := UserConfig{} + err := unmarshalYAMLStrict(contents, &config) return config, err } diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index f784e3f3..ef1ff55a 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "testing" @@ -104,6 +105,73 @@ func TestLoadTopicsFile(t *testing.T) { assert.Equal(t, "topic-test2", topicConfigs[1].Meta.Name) } +func TestLoadUsersFile(t *testing.T) { + userConfigs, err := LoadUsersFile("testdata/test-cluster/users/user-test.yaml") + require.NoError(t, err) + fmt.Println(userConfigs) + assert.Equal(t, 1, len(userConfigs)) + userConfig := userConfigs[0] + + assert.Equal( + t, + UserConfig{ + Meta: UserMeta{ + Name: "user-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + Description: "Test user\n", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Username: "test-user", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: SimpleAuthorization, + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + }, + Operations: []ACLOperation{ + "Read", + "Describe", + }, + }, + { + Resource: ACLResource{ + Type: "group", + Name: "test-group", + PatternType: "prefix", + }, + Operations: []ACLOperation{ + "Read", + }, + }, + }, + }, + }, + }, + userConfig, + ) + assert.NoError(t, userConfig.Validate()) + + // userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-invalid.yaml") + // assert.Equal(t, 1, len(userConfigs)) + // userConfig = userConfigs[0] + // require.NoError(t, err) + // assert.Error(t, userConfig.Validate()) + + // userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-multi.yaml") + // assert.Equal(t, 2, len(userConfigs)) + // assert.Equal(t, "user-test1", userConfigs[0].Meta.Name) + // assert.Equal(t, "user-test2", userConfigs[1].Meta.Name) +} + func TestCheckConsistency(t *testing.T) { os.Setenv("K2_TEST_ENV_VAR", "test-region") defer os.Unsetenv("K2_TEST_ENV_VAR") diff --git a/pkg/util/error.go b/pkg/util/error.go index d721fffb..41cdfb6c 100644 --- a/pkg/util/error.go +++ b/pkg/util/error.go @@ -49,18 +49,3 @@ func AlterPartitionReassignmentsRequestAssignmentError(results []kafka.AlterPart } return nil } - -func DescribeUserScramCredentialsResponseResultsError(results []kafka.DescribeUserScramCredentialsResponseResult) error { - errors := map[string]error{} - var hasErrors bool - for _, result := range results { - if result.Error != nil { - hasErrors = true - errors[result.User] = result.Error - } - } - if hasErrors { - return fmt.Errorf("%+v", errors) - } - return nil -} From 5d46ebe34f12924e4571294ce77f509aa9e87db0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 3 Oct 2023 15:25:04 -0400 Subject: [PATCH 066/116] add uncommitted files --- cmd/topicctl/subcmd/applyuser.go | 171 ++++++++++++++++++ pkg/apply/applyuser.go | 97 ++++++++++ pkg/config/load_test.go | 19 +- .../test-cluster/users/user-test-invalid.yaml | 28 +++ .../test-cluster/users/user-test-multi.yaml | 62 +++++++ .../test-cluster/users/user-test.yaml | 28 +++ pkg/config/user.go | 100 ++++++++++ 7 files changed, 495 insertions(+), 10 deletions(-) create mode 100644 cmd/topicctl/subcmd/applyuser.go create mode 100644 pkg/apply/applyuser.go create mode 100644 pkg/config/testdata/test-cluster/users/user-test-invalid.yaml create mode 100644 pkg/config/testdata/test-cluster/users/user-test-multi.yaml create mode 100644 pkg/config/testdata/test-cluster/users/user-test.yaml create mode 100644 pkg/config/user.go diff --git a/cmd/topicctl/subcmd/applyuser.go b/cmd/topicctl/subcmd/applyuser.go new file mode 100644 index 00000000..6a74d612 --- /dev/null +++ b/cmd/topicctl/subcmd/applyuser.go @@ -0,0 +1,171 @@ +package subcmd + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/apply" + "github.com/segmentio/topicctl/pkg/cli" + "github.com/segmentio/topicctl/pkg/config" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var applyUserCmd = &cobra.Command{ + Use: "apply-user [user configs]", + Short: "apply one or more user configs", + Args: cobra.MinimumNArgs(1), + RunE: applyUserRun, +} + +func init() { + applyUserCmd.Flags().BoolVar( + &applyConfig.dryRun, + "dry-run", + false, + "Do a dry-run", + ) + // TODO: fix this and make it work for users + applyUserCmd.Flags().StringVar( + &applyConfig.pathPrefix, + "path-prefix", + os.Getenv("TOPICCTL_APPLY_PATH_PREFIX"), + "Prefix for topic config paths", + ) + applyUserCmd.Flags().BoolVar( + &applyConfig.skipConfirm, + "skip-confirm", + false, + "Skip confirmation prompts during apply process", + ) + + addSharedConfigOnlyFlags(applyUserCmd, &applyConfig.shared) + RootCmd.AddCommand(applyUserCmd) +} + +func applyUserRun(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + go func() { + <-sigChan + cancel() + }() + + // Keep a cache of the admin clients with the cluster config path as the key + adminClients := map[string]admin.Client{} + + defer func() { + for _, adminClient := range adminClients { + adminClient.Close() + } + }() + + matchCount := 0 + + for _, arg := range args { + if applyConfig.pathPrefix != "" && !filepath.IsAbs(arg) { + arg = filepath.Join(applyConfig.pathPrefix, arg) + } + + matches, err := filepath.Glob(arg) + if err != nil { + return err + } + + for _, match := range matches { + matchCount++ + if err := applyUser(ctx, match, adminClients); err != nil { + return err + } + } + } + + if matchCount == 0 { + return fmt.Errorf("No user configs match the provided args (%+v)", args) + } + + return nil +} + +func applyUser( + ctx context.Context, + userConfigPath string, + adminClients map[string]admin.Client, +) error { + clusterConfigPath, err := clusterConfigForUserApply(userConfigPath) + if err != nil { + return err + } + + userConfigs, err := config.LoadUsersFile(userConfigPath) + if err != nil { + return err + } + + clusterConfig, err := config.LoadClusterFile(clusterConfigPath, applyConfig.shared.expandEnv) + if err != nil { + return err + } + + adminClient, ok := adminClients[clusterConfigPath] + if !ok { + adminClient, err = clusterConfig.NewAdminClient( + ctx, + nil, + applyConfig.dryRun, + applyConfig.shared.saslUsername, + applyConfig.shared.saslPassword, + ) + if err != nil { + return err + } + adminClients[clusterConfigPath] = adminClient + } + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, false) + + for _, userConfig := range userConfigs { + // userConfig.SetDefaults() + log.Infof( + "Processing user %s in config %s with cluster config %s", + userConfig.Meta.Name, + userConfigPath, + clusterConfigPath, + ) + + applierConfig := apply.UserApplierConfig{ + UserConfig: userConfig, + ClusterConfig: clusterConfig, + // TODO: support dryrun and skipconfirm + } + + if err := cliRunner.ApplyUser(ctx, applierConfig); err != nil { + return err + } + } + + return nil +} + +// TODO: move this into a util function shared between this and apply topic +func clusterConfigForUserApply(userConfigPath string) (string, error) { + if applyConfig.shared.clusterConfig != "" { + return applyConfig.shared.clusterConfig, nil + } + + return filepath.Abs( + filepath.Join( + filepath.Dir(userConfigPath), + "..", + "cluster.yaml", + ), + ) +} diff --git a/pkg/apply/applyuser.go b/pkg/apply/applyuser.go new file mode 100644 index 00000000..f48aae3a --- /dev/null +++ b/pkg/apply/applyuser.go @@ -0,0 +1,97 @@ +package apply + +import ( + "context" + "errors" + + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/config" + log "github.com/sirupsen/logrus" +) + +// TODO: dry this up with the apply.go file + +// TODO: are these structs even necessary? +type UserApplierConfig struct { + DryRun bool + SkipConfirm bool + UserConfig config.UserConfig + ClusterConfig config.ClusterConfig +} + +type UserApplier struct { + config UserApplierConfig + adminClient admin.Client + + clusterConfig config.ClusterConfig + userConfig config.UserConfig + userName string +} + +func NewUserApplier( + ctx context.Context, + adminClient admin.Client, + applierConfig UserApplierConfig, +) (*UserApplier, error) { + if !adminClient.GetSupportedFeatures().Applies { + return nil, + errors.New( + "Admin client does not support features needed for apply; please us zk-based client instead.", + ) + } + + return &UserApplier{ + config: applierConfig, + adminClient: adminClient, + clusterConfig: applierConfig.ClusterConfig, + userConfig: applierConfig.UserConfig, + userName: applierConfig.UserConfig.Meta.Name, + }, nil +} + +func (u *UserApplier) Apply(ctx context.Context) error { + log.Info("Validating configs...") + + if err := u.clusterConfig.Validate(); err != nil { + return err + } + + if err := u.userConfig.Validate(); err != nil { + return err + } + + log.Info("Checking if user already exists...") + + userInfo, err := u.adminClient.GetUsers(ctx, []string{u.userName}) + if err != nil { + return err + } + + if len(userInfo) == 0 { + return u.applyNewUser(ctx) + } + // TODO: handle case where this returns multiple users due to multiple creds being created + + return u.applyExistingUser(ctx, userInfo[0]) +} + +func (u *UserApplier) applyNewUser(ctx context.Context) error { + user, err := u.userConfig.ToNewUserScramCredentialsUpsertion() + if err != nil { + return err + } + + if u.config.DryRun { + log.Infof("Would create user with config %+v", user) + return nil + } + return u.adminClient.CreateUser(ctx, user) +} + +func (u *UserApplier) applyExistingUser( + ctx context.Context, + userInfo admin.UserInfo, +) error { + log.Infof("Updating existing user '%s'", u.userName) + return nil +} diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index ef1ff55a..ad898e1b 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -125,7 +125,6 @@ func TestLoadUsersFile(t *testing.T) { Spec: UserSpec{ Authentication: AuthenticationConfig{ Type: "scram-sha-512", - Username: "test-user", Password: "test-password", }, Authorization: AuthorizationConfig{ @@ -160,16 +159,16 @@ func TestLoadUsersFile(t *testing.T) { ) assert.NoError(t, userConfig.Validate()) - // userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-invalid.yaml") - // assert.Equal(t, 1, len(userConfigs)) - // userConfig = userConfigs[0] - // require.NoError(t, err) - // assert.Error(t, userConfig.Validate()) + userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-invalid.yaml") + assert.Equal(t, 1, len(userConfigs)) + userConfig = userConfigs[0] + require.NoError(t, err) + assert.Error(t, userConfig.Validate()) - // userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-multi.yaml") - // assert.Equal(t, 2, len(userConfigs)) - // assert.Equal(t, "user-test1", userConfigs[0].Meta.Name) - // assert.Equal(t, "user-test2", userConfigs[1].Meta.Name) + userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-multi.yaml") + assert.Equal(t, 2, len(userConfigs)) + assert.Equal(t, "user-test1", userConfigs[0].Meta.Name) + assert.Equal(t, "user-test2", userConfigs[1].Meta.Name) } func TestCheckConsistency(t *testing.T) { diff --git a/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml b/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml new file mode 100644 index 00000000..21a1d800 --- /dev/null +++ b/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml @@ -0,0 +1,28 @@ +meta: + name: user-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: invalid + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - Read + - Describe + - resource: + type: group + name: test-group + patternType: prefix + operations: + - Read diff --git a/pkg/config/testdata/test-cluster/users/user-test-multi.yaml b/pkg/config/testdata/test-cluster/users/user-test-multi.yaml new file mode 100644 index 00000000..2c296b97 --- /dev/null +++ b/pkg/config/testdata/test-cluster/users/user-test-multi.yaml @@ -0,0 +1,62 @@ +# This is an empty config. +--- +meta: + name: user-test1 + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: simple + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - Read + - Describe + - resource: + type: group + name: test-group + patternType: prefix + operations: + - Read +--- +meta: + name: user-test2 + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: simple + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - Read + - Describe + - resource: + type: group + name: test-group + patternType: prefix + operations: + - Read +--- +# Another empty one + diff --git a/pkg/config/testdata/test-cluster/users/user-test.yaml b/pkg/config/testdata/test-cluster/users/user-test.yaml new file mode 100644 index 00000000..b818a1fe --- /dev/null +++ b/pkg/config/testdata/test-cluster/users/user-test.yaml @@ -0,0 +1,28 @@ +meta: + name: user-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: simple + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - Read + - Describe + - resource: + type: group + name: test-group + patternType: prefix + operations: + - Read diff --git a/pkg/config/user.go b/pkg/config/user.go new file mode 100644 index 00000000..b942438e --- /dev/null +++ b/pkg/config/user.go @@ -0,0 +1,100 @@ +package config + +import ( + "crypto/rand" + "crypto/sha512" + "fmt" + + "github.com/segmentio/kafka-go" + "github.com/xdg-go/pbkdf2" +) + +// TODO: most of this could be abstracted away and made available for any resource + +type UserConfig struct { + Meta UserMeta `json:"meta"` + Spec UserSpec `json:"spec"` +} + +// UserMeta stores the (mostly immutable) metadata associated with a topic. +// Inspired by the meta structs in Kubernetes objects. +type UserMeta struct { + Name string `json:"name"` + Cluster string `json:"cluster"` + Region string `json:"region"` + Environment string `json:"environment"` + Description string `json:"description"` + Labels map[string]string `json:"labels"` +} + +type UserSpec struct { + Authentication AuthenticationConfig `json:"authentication"` + Authorization AuthorizationConfig `json:"authorization,omitempty"` +} + +type AuthenticationConfig struct { + // TODO: extend this type to capture future types + Type string `json:"type"` + // TODO: extend this to a type that supports SSMRef + Password string `json:"password"` +} + +type AuthorizationConfig struct { + Type AuthorizationType `json:"type"` + ACLs []ACL `json:"acls,omitempty"` +} + +type AuthorizationType string + +type ACL struct { + Resource ACLResource `json:"resource"` + Operations []ACLOperation `json:"operations"` +} + +type ACLResource struct { + Type string `json:"type"` + Name string `json:"name"` + PatternType string `json:"patternType"` + Principal string `json:"principal"` + Host string `json:"host"` +} + +type ACLOperation string + +const ( + SimpleAuthorization AuthorizationType = "simple" +) + +// TODO: add validation +// maybe consider breaking out common functionality for Meta validation +func (u *UserConfig) Validate() error { + // TODO: validation authenticationtype + + // TODO: validate all enums + var err error + + return err +} + +const ( + // Currently only scram-sha-512 is supported + ScramMechanism kafka.ScramMechanism = kafka.ScramMechanismSha512 + // Use the same default as Postgres and Strimzi for Scram iterations + ScramIterations int = 4096 +) + +func (u UserConfig) ToNewUserScramCredentialsUpsertion() (kafka.UserScramCredentialsUpsertion, error) { + salt := make([]byte, 24) + if _, err := rand.Read(salt); err != nil { + return kafka.UserScramCredentialsUpsertion{}, fmt.Errorf("User %s: unable to generate salt: %v", u.Meta.Name, err) + } + saltedPassword := pbkdf2.Key([]byte(u.Spec.Authentication.Password), salt, ScramIterations, sha512.Size, sha512.New) + + return kafka.UserScramCredentialsUpsertion{ + Name: u.Meta.Name, + Mechanism: ScramMechanism, + Iterations: ScramIterations, + Salt: salt, + SaltedPassword: saltedPassword, + }, nil +} From 0f8d28393a0e4964bc1df0554072eb32414d13cf Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 4 Oct 2023 11:54:09 -0400 Subject: [PATCH 067/116] start adding validation --- pkg/admin/types.go | 16 +-- pkg/config/load_test.go | 14 +-- .../test-cluster/users/user-test.yaml | 8 +- pkg/config/user.go | 103 ++++++++++++++++-- 4 files changed, 111 insertions(+), 30 deletions(-) diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 73861a06..36e9673f 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -93,7 +93,7 @@ type ACLInfo struct { // as a Cobra flag. type ResourceType kafka.ResourceType -var resourceTypeMap = map[string]kafka.ResourceType{ +var ResourceTypeMap = map[string]kafka.ResourceType{ "any": kafka.ResourceTypeAny, "topic": kafka.ResourceTypeTopic, "group": kafka.ResourceTypeGroup, @@ -124,7 +124,7 @@ func (r *ResourceType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (r *ResourceType) Set(v string) error { - rt, ok := resourceTypeMap[strings.ToLower(v)] + rt, ok := ResourceTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) } @@ -143,7 +143,7 @@ func (r *ResourceType) Type() string { // as a Cobra flag. type PatternType kafka.PatternType -var patternTypeMap = map[string]kafka.PatternType{ +var PatternTypeMap = map[string]kafka.PatternType{ "any": kafka.PatternTypeAny, "match": kafka.PatternTypeMatch, "literal": kafka.PatternTypeLiteral, @@ -168,7 +168,7 @@ func (p *PatternType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (p *PatternType) Set(v string) error { - pt, ok := patternTypeMap[strings.ToLower(v)] + pt, ok := PatternTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "match", "literal", or "prefixed"`) } @@ -187,7 +187,7 @@ func (r *PatternType) Type() string { // as a Cobra flag. type ACLOperationType kafka.ACLOperationType -var aclOperationTypeMap = map[string]kafka.ACLOperationType{ +var AclOperationTypeMap = map[string]kafka.ACLOperationType{ "any": kafka.ACLOperationTypeAny, "all": kafka.ACLOperationTypeAll, "read": kafka.ACLOperationTypeRead, @@ -236,7 +236,7 @@ func (o *ACLOperationType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (o *ACLOperationType) Set(v string) error { - ot, ok := aclOperationTypeMap[strings.ToLower(v)] + ot, ok := AclOperationTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`) } @@ -255,7 +255,7 @@ func (o *ACLOperationType) Type() string { // as a Cobra flag. type ACLPermissionType kafka.ACLPermissionType -var aclPermissionTypeMap = map[string]kafka.ACLPermissionType{ +var AclPermissionTypeMap = map[string]kafka.ACLPermissionType{ "any": kafka.ACLPermissionTypeAny, "allow": kafka.ACLPermissionTypeAllow, "deny": kafka.ACLPermissionTypeDeny, @@ -277,7 +277,7 @@ func (p *ACLPermissionType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (p *ACLPermissionType) Set(v string) error { - pt, ok := aclPermissionTypeMap[strings.ToLower(v)] + pt, ok := AclPermissionTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "allow", or "deny"`) } diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index ad898e1b..aeb8eab3 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -128,7 +128,7 @@ func TestLoadUsersFile(t *testing.T) { Password: "test-password", }, Authorization: AuthorizationConfig{ - Type: SimpleAuthorization, + Type: "simple", ACLs: []ACL{ { Resource: ACLResource{ @@ -136,19 +136,19 @@ func TestLoadUsersFile(t *testing.T) { Name: "test-topic", PatternType: "literal", }, - Operations: []ACLOperation{ - "Read", - "Describe", + Operations: []string{ + "read", + "describe", }, }, { Resource: ACLResource{ Type: "group", Name: "test-group", - PatternType: "prefix", + PatternType: "prefixed", }, - Operations: []ACLOperation{ - "Read", + Operations: []string{ + "read", }, }, }, diff --git a/pkg/config/testdata/test-cluster/users/user-test.yaml b/pkg/config/testdata/test-cluster/users/user-test.yaml index b818a1fe..a75ea237 100644 --- a/pkg/config/testdata/test-cluster/users/user-test.yaml +++ b/pkg/config/testdata/test-cluster/users/user-test.yaml @@ -18,11 +18,11 @@ spec: name: test-topic patternType: literal operations: - - Read - - Describe + - read + - describe - resource: type: group name: test-group - patternType: prefix + patternType: prefixed operations: - - Read + - read diff --git a/pkg/config/user.go b/pkg/config/user.go index b942438e..79e1cc1d 100644 --- a/pkg/config/user.go +++ b/pkg/config/user.go @@ -5,7 +5,9 @@ import ( "crypto/sha512" "fmt" + "github.com/hashicorp/go-multierror" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/xdg-go/pbkdf2" ) @@ -33,12 +35,21 @@ type UserSpec struct { } type AuthenticationConfig struct { - // TODO: extend this type to capture future types - Type string `json:"type"` + Type AuthenticationType `json:"type"` // TODO: extend this to a type that supports SSMRef Password string `json:"password"` } +type AuthenticationType string + +const ( + ScramSha512 AuthenticationType = "scram-sha-512" +) + +var allAuthenticationTypes = []AuthenticationType{ + ScramSha512, +} + type AuthorizationConfig struct { Type AuthorizationType `json:"type"` ACLs []ACL `json:"acls,omitempty"` @@ -46,9 +57,17 @@ type AuthorizationConfig struct { type AuthorizationType string +const ( + SimpleAuthorization AuthorizationType = "simple" +) + +var allAuthorizationTypes = []AuthorizationType{ + SimpleAuthorization, +} + type ACL struct { - Resource ACLResource `json:"resource"` - Operations []ACLOperation `json:"operations"` + Resource ACLResource `json:"resource"` + Operations []string `json:"operations"` } type ACLResource struct { @@ -59,20 +78,82 @@ type ACLResource struct { Host string `json:"host"` } -type ACLOperation string +func keys[K comparable, V any](m map[K]V) []K { + keys := make([]K, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} -const ( - SimpleAuthorization AuthorizationType = "simple" -) +var allResourceTypes = keys(admin.ResourceTypeMap) +var allPatternTypes = keys(admin.PatternTypeMap) +var allOperationTypes = keys(admin.AclOperationTypeMap) + +func (u *UserConfig) SetDefaults() { + if u.Spec.Authorization.Type == "" { + u.Spec.Authorization.Type = SimpleAuthorization + } +} // TODO: add validation // maybe consider breaking out common functionality for Meta validation func (u *UserConfig) Validate() error { - // TODO: validation authenticationtype - - // TODO: validate all enums + // TODO: validate meta + // TODO: validate password types var err error + authenticationTypeFound := false + for _, authenticationType := range allAuthenticationTypes { + if authenticationType == u.Spec.Authentication.Type { + authenticationTypeFound = true + } + } + + if !authenticationTypeFound { + err = multierror.Append( + err, + fmt.Errorf("Authentication Type must be in %+v", allAuthenticationTypes), + ) + } + + authorizationTypeFound := false + for _, authorizationType := range allAuthorizationTypes { + if authorizationType == u.Spec.Authorization.Type { + authorizationTypeFound = true + } + } + + if !authorizationTypeFound { + err = multierror.Append( + err, + fmt.Errorf("Authorization Type must be in %+v", allAuthorizationTypes), + ) + } + + for _, acl := range u.Spec.Authorization.ACLs { + if _, ok := admin.ResourceTypeMap[acl.Resource.Type]; !ok { + err = multierror.Append( + err, + fmt.Errorf("ACL Resource Type must be in %+v", allResourceTypes), + ) + } + if _, ok := admin.PatternTypeMap[acl.Resource.PatternType]; !ok { + err = multierror.Append( + err, + fmt.Errorf("ACL Resource PatternType must be in %+v", allPatternTypes), + ) + } + for _, operation := range acl.Operations { + if _, ok := admin.AclOperationTypeMap[operation]; !ok { + err = multierror.Append( + err, + fmt.Errorf("ACL OperationType must be in %+v", allOperationTypes), + ) + } + } + } + return err } From f7cd4f8aa90d08fc1afcdeb40cb45872feceeb45 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 4 Oct 2023 16:04:21 -0400 Subject: [PATCH 068/116] meta validation for users --- pkg/config/topic.go | 1 + pkg/config/user.go | 19 ++- pkg/config/user_test.go | 288 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 303 insertions(+), 5 deletions(-) create mode 100644 pkg/config/user_test.go diff --git a/pkg/config/topic.go b/pkg/config/topic.go index 5b7240cb..935e4af8 100644 --- a/pkg/config/topic.go +++ b/pkg/config/topic.go @@ -193,6 +193,7 @@ func (t TopicConfig) Validate(numRacks int) error { if t.Meta.Environment == "" { err = multierror.Append(err, errors.New("Environment must be set")) } + if t.Spec.Partitions <= 0 { err = multierror.Append(err, errors.New("Partitions must be a positive number")) } diff --git a/pkg/config/user.go b/pkg/config/user.go index 79e1cc1d..076f0d79 100644 --- a/pkg/config/user.go +++ b/pkg/config/user.go @@ -3,6 +3,7 @@ package config import ( "crypto/rand" "crypto/sha512" + "errors" "fmt" "github.com/hashicorp/go-multierror" @@ -18,8 +19,6 @@ type UserConfig struct { Spec UserSpec `json:"spec"` } -// UserMeta stores the (mostly immutable) metadata associated with a topic. -// Inspired by the meta structs in Kubernetes objects. type UserMeta struct { Name string `json:"name"` Cluster string `json:"cluster"` @@ -96,13 +95,23 @@ func (u *UserConfig) SetDefaults() { } } -// TODO: add validation -// maybe consider breaking out common functionality for Meta validation func (u *UserConfig) Validate() error { - // TODO: validate meta // TODO: validate password types var err error + if u.Meta.Name == "" { + err = multierror.Append(err, errors.New("Name must be set")) + } + if u.Meta.Cluster == "" { + err = multierror.Append(err, errors.New("Cluster must be set")) + } + if u.Meta.Region == "" { + err = multierror.Append(err, errors.New("Region must be set")) + } + if u.Meta.Environment == "" { + err = multierror.Append(err, errors.New("Environment must be set")) + } + authenticationTypeFound := false for _, authenticationType := range allAuthenticationTypes { if authenticationType == u.Spec.Authentication.Type { diff --git a/pkg/config/user_test.go b/pkg/config/user_test.go new file mode 100644 index 00000000..626da78f --- /dev/null +++ b/pkg/config/user_test.go @@ -0,0 +1,288 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUserValidate(t *testing.T) { + type testCase struct { + description string + userConfig UserConfig + expError bool + } + + testCases := []testCase{ + { + description: "happy path", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "simple", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + }, + expError: false, + }, + { + description: "missing meta fields", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "simple", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "invalid authentication type", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "invalid", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "simple", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "invalid authorization type", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "invalid", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "invalid ACL resource type", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "simple", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "invalid", + Name: "test-topic", + PatternType: "literal", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "invalid ACL resource pattern type", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "simple", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "invalid", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "invalid ACL operation type", + userConfig: UserConfig{ + Meta: UserMeta{ + Name: "test-user", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + Spec: UserSpec{ + Authentication: AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: AuthorizationConfig{ + Type: "simple", + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + Principal: "User:alice", + Host: "*", + }, + Operations: []string{ + "invalid", + "describe", + }, + }, + }, + }, + }, + }, + expError: true, + }, + } + + for _, testCase := range testCases { + err := testCase.userConfig.Validate() + if testCase.expError { + assert.Error(t, err, testCase.description) + } else { + assert.NoError(t, err, testCase.description) + } + } +} + +func TestUserConfigFromUserInfo(t *testing.T) { + t.Fatal("implement me") +} From 01efe00041ddf9a5b300bb2cbd087277196b389f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 10 Oct 2023 10:17:18 -0400 Subject: [PATCH 069/116] wip --- cmd/topicctl/subcmd/applyuser.go | 2 + pkg/admin/brokerclient.go | 10 +- pkg/admin/brokerclient_test.go | 20 +-- pkg/admin/client.go | 6 +- pkg/admin/zkclient.go | 4 +- pkg/admin/zkclient_test.go | 2 +- pkg/apply/applyuser.go | 96 +++++++++++-- pkg/apply/applyuser_test.go | 135 ++++++++++++++++++ pkg/apply/format.go | 12 ++ pkg/config/load_test.go | 20 +-- .../test-cluster/users/user-test-invalid.yaml | 6 +- .../test-cluster/users/user-test-multi.yaml | 16 +-- pkg/config/user.go | 40 +++++- pkg/config/user_test.go | 6 + 14 files changed, 315 insertions(+), 60 deletions(-) create mode 100644 pkg/apply/applyuser_test.go diff --git a/cmd/topicctl/subcmd/applyuser.go b/cmd/topicctl/subcmd/applyuser.go index 6a74d612..72f9116d 100644 --- a/cmd/topicctl/subcmd/applyuser.go +++ b/cmd/topicctl/subcmd/applyuser.go @@ -142,6 +142,8 @@ func applyUser( ) applierConfig := apply.UserApplierConfig{ + DryRun: applyConfig.dryRun, + SkipConfirm: applyConfig.skipConfirm, UserConfig: userConfig, ClusterConfig: clusterConfig, // TODO: support dryrun and skipconfirm diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index e616fc1d..cfe66bb9 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -806,19 +806,17 @@ func (c *BrokerAdminClient) GetACLs( return aclinfos, nil } -// CreateACL creates an ACL in the cluster. -func (c *BrokerAdminClient) CreateACL( +// CreateACLs creates ACLs in the cluster. +func (c *BrokerAdminClient) CreateACLs( ctx context.Context, - entry kafka.ACLEntry, + acls []kafka.ACLEntry, ) error { if c.config.ReadOnly { return errors.New("Cannot create ACL in read-only mode") } req := kafka.CreateACLsRequest{ - ACLs: []kafka.ACLEntry{ - entry, - }, + ACLs: acls, } log.Debugf("CreateACLs request: %+v", req) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 5d6103a4..5f8cfce2 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -616,16 +616,18 @@ func TestBrokerClientCreateGetACL(t *testing.T) { } }() - err = client.CreateACL( + err = client.CreateACLs( ctx, - kafka.ACLEntry{ - Principal: principal, - PermissionType: kafka.ACLPermissionTypeAllow, - Operation: kafka.ACLOperationTypeRead, - ResourceType: kafka.ResourceTypeTopic, - ResourcePatternType: kafka.PatternTypeLiteral, - ResourceName: topicName, - Host: "*", + []kafka.ACLEntry{ + { + Principal: principal, + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + ResourceType: kafka.ResourceTypeTopic, + ResourcePatternType: kafka.PatternTypeLiteral, + ResourceName: topicName, + Host: "*", + }, }, ) require.NoError(t, err) diff --git a/pkg/admin/client.go b/pkg/admin/client.go index d94bb46b..7ffe53c9 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -74,10 +74,10 @@ type Client interface { config kafka.TopicConfig, ) error - // Create ACL creates an ACL in the cluster. - CreateACL( + // CreateACLs creates ACLs in the cluster. + CreateACLs( ctx context.Context, - entry kafka.ACLEntry, + acls []kafka.ACLEntry, ) error // CreateUser creates a user in zookeeper. diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 9a93b997..9142ef73 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -429,9 +429,9 @@ func (c *ZKAdminClient) GetACLs( return nil, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } -func (c *ZKAdminClient) CreateACL( +func (c *ZKAdminClient) CreateACLs( ctx context.Context, - entry kafka.ACLEntry, + acls []kafka.ACLEntry, ) error { return errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 8c87e1ab..cb537114 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1103,7 +1103,7 @@ func TestZkCreateACL(t *testing.T) { require.NoError(t, err) defer adminClient.Close() - err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) + err = adminClient.CreateACLs(ctx, []kafka.ACLEntry{}) assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } diff --git a/pkg/apply/applyuser.go b/pkg/apply/applyuser.go index f48aae3a..a5b50319 100644 --- a/pkg/apply/applyuser.go +++ b/pkg/apply/applyuser.go @@ -3,7 +3,9 @@ package apply import ( "context" "errors" + "fmt" + "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/config" log "github.com/sirupsen/logrus" @@ -25,7 +27,6 @@ type UserApplier struct { clusterConfig config.ClusterConfig userConfig config.UserConfig - userName string } func NewUserApplier( @@ -45,7 +46,6 @@ func NewUserApplier( adminClient: adminClient, clusterConfig: applierConfig.ClusterConfig, userConfig: applierConfig.UserConfig, - userName: applierConfig.UserConfig.Meta.Name, }, nil } @@ -53,45 +53,117 @@ func (u *UserApplier) Apply(ctx context.Context) error { log.Info("Validating configs...") if err := u.clusterConfig.Validate(); err != nil { - return err + return fmt.Errorf("error validating cluster config: %v", err) } if err := u.userConfig.Validate(); err != nil { - return err + return fmt.Errorf("error validating user config: %v", err) } log.Info("Checking if user already exists...") - userInfo, err := u.adminClient.GetUsers(ctx, []string{u.userName}) + userInfo, err := u.adminClient.GetUsers(ctx, []string{u.userConfig.Meta.Name}) if err != nil { - return err + return fmt.Errorf("error checking if user already exists: %v", err) } if len(userInfo) == 0 { - return u.applyNewUser(ctx) + err = u.applyNewUser(ctx) + } else { + // TODO: handle case where this returns multiple users due to multiple creds being created + err = u.applyExistingUser(ctx, userInfo[0]) } - // TODO: handle case where this returns multiple users due to multiple creds being created - return u.applyExistingUser(ctx, userInfo[0]) + if err != nil { + return fmt.Errorf("error applying existing user: %v", err) + } + + log.Info("Checking if ACLs already exist for this user...") + + existingACLs, err := u.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeAny, + ResourcePatternTypeFilter: kafka.PatternTypeAny, + PrincipalFilter: fmt.Sprintf("User:%s", u.userConfig.Meta.Name), + Operation: kafka.ACLOperationTypeAny, + PermissionType: kafka.ACLPermissionTypeAny, + }) + + if err != nil { + return fmt.Errorf("error checking existing ACLs for user %s: %v", u.userConfig.Meta.Name, err) + } + + log.Info("Found ", len(existingACLs), " existing ACLs: ", existingACLs) + + // convert existingACLs from []kafka.ACLInfo to []kafka.ACLEntry + existingACLEntries := []kafka.ACLEntry{} + for _, acl := range existingACLs { + existingACLEntries = append(existingACLEntries, kafka.ACLEntry{ + ResourceType: kafka.ResourceType(acl.ResourceType), + ResourceName: acl.ResourceName, + ResourcePatternType: kafka.PatternType(acl.PatternType), + Principal: acl.Principal, + Host: acl.Host, + Operation: kafka.ACLOperationType(acl.Operation), + PermissionType: kafka.ACLPermissionType(acl.PermissionType), + }) + } + + // TODO: find orphaned ACLs, new ACLs, existing ACLs + + acls := u.userConfig.ToNewACLEntries() + + // Compare acls and existingACLEntries + + if len(acls) == 0 { + return nil + } + + log.Infof("Creating new ACLs for user with config %+v", acls) + + err = u.adminClient.CreateACLs(ctx, acls) + if err != nil { + return fmt.Errorf("error creating new ACLs: %v", err) + } + return nil } func (u *UserApplier) applyNewUser(ctx context.Context) error { user, err := u.userConfig.ToNewUserScramCredentialsUpsertion() if err != nil { - return err + return fmt.Errorf("error creating UserScramCredentialsUpsertion: %v", err) } if u.config.DryRun { log.Infof("Would create user with config %+v", user) + // TODO: dry run acls as well return nil } - return u.adminClient.CreateUser(ctx, user) + + log.Infof( + "It looks like this user doesn't already exists. Will create it with this config:\n%s", + FormatNewUserConfig(user), + ) + + ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) + if !ok { + return errors.New("Stopping because of user response") + } + + log.Infof("Creating new user with config %+v", user) + + err = u.adminClient.CreateUser(ctx, user) + if err != nil { + return fmt.Errorf("error creating new user: %v", err) + } + + return nil } +// TODO: support this func (u *UserApplier) applyExistingUser( ctx context.Context, userInfo admin.UserInfo, ) error { - log.Infof("Updating existing user '%s'", u.userName) + log.Infof("Updating existing user: %s", u.userConfig.Meta.Name) return nil } diff --git a/pkg/apply/applyuser_test.go b/pkg/apply/applyuser_test.go new file mode 100644 index 00000000..920775b0 --- /dev/null +++ b/pkg/apply/applyuser_test.go @@ -0,0 +1,135 @@ +package apply + +import ( + "context" + "testing" + "time" + + "github.com/segmentio/topicctl/pkg/config" + "github.com/segmentio/topicctl/pkg/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TODO: write these tests +func TestApplyNewUser(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + userName := util.RandomString("apply-user-", 6) + userConfig := config.UserConfig{ + Meta: config.UserMeta{ + Name: userName, + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.UserSpec{ + Authentication: config.AuthenticationConfig{ + Type: "scram-sha-512", + Password: "test-password", + }, + Authorization: config.AuthorizationConfig{ + Type: "simple", + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: "topic", + Name: "test-topic", + PatternType: "literal", + Host: "*", + }, + Operations: []string{ + "read", + "describe", + }, + }, + }, + }, + }, + } + + applier := testUserApplier(ctx, t, userConfig) + + defer applier.adminClient.Close() + err := applier.Apply(ctx) + require.NoError(t, err) + + userInfo, err := applier.adminClient.GetUsers(ctx, []string{userName}) + require.NoError(t, err) + assert.Equal(t, 1, len(userInfo)) + assert.Equal(t, userName, userInfo[0].Name) + + // TODO: login with user creds + clusterConfig := config.ClusterConfig{ + Meta: config.ClusterMeta{ + Name: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + + Spec: config.ClusterSpec{ + BootstrapAddrs: []string{util.TestKafkaAddr()}, + SASL: config.SASLConfig{ + Enabled: true, + Mechanism: "scram-sha-512", + Username: userName, + Password: "test-password", + }, + }, + } + + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") + require.NoError(t, err) + _, err = adminClient.GetUsers(ctx, []string{userName}) + require.NoError(t, err) + // TODO: check acls + // TODO: check empty host gets coalesced to "*" +} + +func TestApplyExistingUser(t *testing.T) {} + +func TestApplyUserDryRun(t *testing.T) {} + +func testUserApplier( + ctx context.Context, + t *testing.T, + userConfig config.UserConfig, +) *UserApplier { + clusterConfig := config.ClusterConfig{ + Meta: config.ClusterMeta{ + Name: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + + Spec: config.ClusterSpec{ + BootstrapAddrs: []string{util.TestKafkaAddr()}, + //ZKAddrs: []string{util.TestZKAddr()}, + ZKLockPath: "/topicctl/locks", + }, + } + + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") + require.NoError(t, err) + + applier, err := NewUserApplier( + ctx, + adminClient, + UserApplierConfig{ + ClusterConfig: clusterConfig, + UserConfig: userConfig, + DryRun: false, + SkipConfirm: true, + }, + ) + require.NoError(t, err) + return applier +} + +// create a function that lists all users and deletes them +func TestDeleteUser(t *testing.T) { + // create a user + + // delete the user +} diff --git a/pkg/apply/format.go b/pkg/apply/format.go index 2cc6361f..8c3725ce 100644 --- a/pkg/apply/format.go +++ b/pkg/apply/format.go @@ -25,6 +25,18 @@ func FormatNewTopicConfig(config kafka.TopicConfig) string { return string(content) } +// FormatNewUserConfig generates a pretty string representation of a kafka-go +// user config. +func FormatNewUserConfig(config kafka.UserScramCredentialsUpsertion) string { + content, err := json.MarshalIndent(config, "", " ") + if err != nil { + log.Warnf("Error marshalling user config: %+v", err) + return "Error" + } + + return string(content) +} + // FormatSettingsDiff generates a table that summarizes the differences between // the topic settings from a topic config and the settings from ZK. func FormatSettingsDiff( diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index aeb8eab3..f5a599bc 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -159,16 +159,18 @@ func TestLoadUsersFile(t *testing.T) { ) assert.NoError(t, userConfig.Validate()) - userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-invalid.yaml") - assert.Equal(t, 1, len(userConfigs)) - userConfig = userConfigs[0] + invalidUserConfigs, err := LoadUsersFile("testdata/test-cluster/users/user-test-invalid.yaml") + assert.Equal(t, 1, len(invalidUserConfigs)) + invalidUserConfig := invalidUserConfigs[0] require.NoError(t, err) - assert.Error(t, userConfig.Validate()) - - userConfigs, err = LoadUsersFile("testdata/test-cluster/users/user-test-multi.yaml") - assert.Equal(t, 2, len(userConfigs)) - assert.Equal(t, "user-test1", userConfigs[0].Meta.Name) - assert.Equal(t, "user-test2", userConfigs[1].Meta.Name) + assert.Error(t, invalidUserConfig.Validate()) + + multiUserConfigs, err := LoadUsersFile("testdata/test-cluster/users/user-test-multi.yaml") + assert.Equal(t, 2, len(multiUserConfigs)) + assert.Equal(t, "user-test1", multiUserConfigs[0].Meta.Name) + assert.Equal(t, "user-test2", multiUserConfigs[1].Meta.Name) + assert.NoError(t, multiUserConfigs[0].Validate()) + assert.NoError(t, multiUserConfigs[1].Validate()) } func TestCheckConsistency(t *testing.T) { diff --git a/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml b/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml index 21a1d800..8d2c5601 100644 --- a/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml +++ b/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml @@ -18,11 +18,11 @@ spec: name: test-topic patternType: literal operations: - - Read - - Describe + - read + - describe - resource: type: group name: test-group patternType: prefix operations: - - Read + - read diff --git a/pkg/config/testdata/test-cluster/users/user-test-multi.yaml b/pkg/config/testdata/test-cluster/users/user-test-multi.yaml index 2c296b97..6104e23a 100644 --- a/pkg/config/testdata/test-cluster/users/user-test-multi.yaml +++ b/pkg/config/testdata/test-cluster/users/user-test-multi.yaml @@ -20,14 +20,14 @@ spec: name: test-topic patternType: literal operations: - - Read - - Describe + - read + - describe - resource: type: group name: test-group - patternType: prefix + patternType: prefixed operations: - - Read + - read --- meta: name: user-test2 @@ -49,14 +49,14 @@ spec: name: test-topic patternType: literal operations: - - Read - - Describe + - read + - describe - resource: type: group name: test-group - patternType: prefix + patternType: prefixed operations: - - Read + - read --- # Another empty one diff --git a/pkg/config/user.go b/pkg/config/user.go index 076f0d79..77ee5be3 100644 --- a/pkg/config/user.go +++ b/pkg/config/user.go @@ -12,8 +12,6 @@ import ( "github.com/xdg-go/pbkdf2" ) -// TODO: most of this could be abstracted away and made available for any resource - type UserConfig struct { Meta UserMeta `json:"meta"` Spec UserSpec `json:"spec"` @@ -69,6 +67,8 @@ type ACL struct { Operations []string `json:"operations"` } +// TODO: how should principal and permission type be handled? +// principal will always be the meta name and permission type will always be allowed type ACLResource struct { Type string `json:"type"` Name string `json:"name"` @@ -122,7 +122,7 @@ func (u *UserConfig) Validate() error { if !authenticationTypeFound { err = multierror.Append( err, - fmt.Errorf("Authentication Type must be in %+v", allAuthenticationTypes), + fmt.Errorf("Authentication Type must be in %+v, got: %s", allAuthenticationTypes, u.Spec.Authentication.Type), ) } @@ -136,7 +136,7 @@ func (u *UserConfig) Validate() error { if !authorizationTypeFound { err = multierror.Append( err, - fmt.Errorf("Authorization Type must be in %+v", allAuthorizationTypes), + fmt.Errorf("Authorization Type must be in %+v, got: %s", allAuthorizationTypes, u.Spec.Authorization.Type), ) } @@ -144,20 +144,20 @@ func (u *UserConfig) Validate() error { if _, ok := admin.ResourceTypeMap[acl.Resource.Type]; !ok { err = multierror.Append( err, - fmt.Errorf("ACL Resource Type must be in %+v", allResourceTypes), + fmt.Errorf("ACL Resource Type must be in %+v, got: %s", allResourceTypes, acl.Resource.Type), ) } if _, ok := admin.PatternTypeMap[acl.Resource.PatternType]; !ok { err = multierror.Append( err, - fmt.Errorf("ACL Resource PatternType must be in %+v", allPatternTypes), + fmt.Errorf("ACL Resource PatternType must be in %+v, got: %s", allPatternTypes, acl.Resource.PatternType), ) } for _, operation := range acl.Operations { if _, ok := admin.AclOperationTypeMap[operation]; !ok { err = multierror.Append( err, - fmt.Errorf("ACL OperationType must be in %+v", allOperationTypes), + fmt.Errorf("ACL OperationType must be in %+v, got: %s", allOperationTypes, operation), ) } } @@ -188,3 +188,29 @@ func (u UserConfig) ToNewUserScramCredentialsUpsertion() (kafka.UserScramCredent SaltedPassword: saltedPassword, }, nil } + +func (u UserConfig) ToNewACLEntries() []kafka.ACLEntry { + acls := []kafka.ACLEntry{} + + for _, acl := range u.Spec.Authorization.ACLs { + // Data has already been validated before calling this function so no need to check validity + + resourceType, _ := admin.ResourceTypeMap[acl.Resource.Type] + resourcePatternType, _ := admin.PatternTypeMap[acl.Resource.PatternType] + + for _, operation := range acl.Operations { + aclOperation, _ := admin.AclOperationTypeMap[operation] + + acls = append(acls, kafka.ACLEntry{ + ResourceType: resourceType, + ResourceName: acl.Resource.Name, + ResourcePatternType: resourcePatternType, + Principal: fmt.Sprintf("User:%s", u.Meta.Name), + Host: acl.Resource.Host, + Operation: aclOperation, + PermissionType: kafka.ACLPermissionTypeAllow, + }) + } + } + return acls +} diff --git a/pkg/config/user_test.go b/pkg/config/user_test.go index 626da78f..283d45ad 100644 --- a/pkg/config/user_test.go +++ b/pkg/config/user_test.go @@ -283,6 +283,12 @@ func TestUserValidate(t *testing.T) { } } +// TODO: write this test func TestUserConfigFromUserInfo(t *testing.T) { t.Fatal("implement me") } + +// TODO: write this test +func TestACLEntryFromUser(t *testing.T) { + t.Fatal("implement me") +} From 79d9d3bb3f71b710bed0aff15e54c9c94fa9932f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 18 Oct 2023 16:33:35 -0400 Subject: [PATCH 070/116] support dry run and skip confirm --- cmd/topicctl/subcmd/applyuser.go | 1 - pkg/apply/applyuser.go | 22 ++++++++++++++++++---- pkg/config/user.go | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cmd/topicctl/subcmd/applyuser.go b/cmd/topicctl/subcmd/applyuser.go index d5f6aaf4..ca00f8be 100644 --- a/cmd/topicctl/subcmd/applyuser.go +++ b/cmd/topicctl/subcmd/applyuser.go @@ -145,7 +145,6 @@ func applyUser( SkipConfirm: applyConfig.skipConfirm, UserConfig: userConfig, ClusterConfig: clusterConfig, - // TODO: support dryrun and skipconfirm } if err := cliRunner.ApplyUser(ctx, applierConfig); err != nil { diff --git a/pkg/apply/applyuser.go b/pkg/apply/applyuser.go index a5b50319..2ad2fed1 100644 --- a/pkg/apply/applyuser.go +++ b/pkg/apply/applyuser.go @@ -37,7 +37,7 @@ func NewUserApplier( if !adminClient.GetSupportedFeatures().Applies { return nil, errors.New( - "Admin client does not support features needed for apply; please us zk-based client instead.", + "Admin client does not support features needed for apply; You need to upgrade to Kafka version >2.7.0.", ) } @@ -94,7 +94,6 @@ func (u *UserApplier) Apply(ctx context.Context) error { log.Info("Found ", len(existingACLs), " existing ACLs: ", existingACLs) - // convert existingACLs from []kafka.ACLInfo to []kafka.ACLEntry existingACLEntries := []kafka.ACLEntry{} for _, acl := range existingACLs { existingACLEntries = append(existingACLEntries, kafka.ACLEntry{ @@ -118,6 +117,22 @@ func (u *UserApplier) Apply(ctx context.Context) error { return nil } + if u.config.DryRun { + log.Infof("Would create ACLs with config %+v", acls) + return nil + } + + log.Infof( + "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", + // TODO: pretty format these ACLs + acls, + ) + + ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) + if !ok { + return errors.New("Stopping because of user response") + } + log.Infof("Creating new ACLs for user with config %+v", acls) err = u.adminClient.CreateACLs(ctx, acls) @@ -135,7 +150,6 @@ func (u *UserApplier) applyNewUser(ctx context.Context) error { if u.config.DryRun { log.Infof("Would create user with config %+v", user) - // TODO: dry run acls as well return nil } @@ -151,7 +165,7 @@ func (u *UserApplier) applyNewUser(ctx context.Context) error { log.Infof("Creating new user with config %+v", user) - err = u.adminClient.CreateUser(ctx, user) + err = u.adminClient.UpsertUser(ctx, user) if err != nil { return fmt.Errorf("error creating new user: %v", err) } diff --git a/pkg/config/user.go b/pkg/config/user.go index 77ee5be3..888feb8a 100644 --- a/pkg/config/user.go +++ b/pkg/config/user.go @@ -27,7 +27,7 @@ type UserMeta struct { } type UserSpec struct { - Authentication AuthenticationConfig `json:"authentication"` + Authentication AuthenticationConfig `json:"authentication,omitempty"` Authorization AuthorizationConfig `json:"authorization,omitempty"` } From a335872bf59d855cc55553414f10398410ec74fe Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 30 Oct 2023 14:07:48 -0400 Subject: [PATCH 071/116] wip --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/apply/applyuser.go | 24 ++++++------------------ pkg/apply/format.go | 12 ++++++++++++ pkg/config/user.go | 19 +++++++++++++++++++ 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 083f473e..cae7ae6e 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,13 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.44 + github.com/segmentio/kafka-go v0.4.45-0.20231030174323-c6378c391a97 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 + github.com/xdg-go/pbkdf2 v1.0.0 golang.org/x/crypto v0.14.0 ) @@ -37,7 +38,6 @@ require ( github.com/pkg/term v0.0.0-20200520122047-c3ffed290a03 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/go.sum b/go.sum index f20d07f4..da752a0f 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= -github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4= -github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/segmentio/kafka-go v0.4.45-0.20231030174323-c6378c391a97 h1:vKYoioQZ7SgGcES2pKoNq7zV8ncKNvblHp+0O+dOeI0= +github.com/segmentio/kafka-go v0.4.45-0.20231030174323-c6378c391a97/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= diff --git a/pkg/apply/applyuser.go b/pkg/apply/applyuser.go index 2ad2fed1..e3b99df4 100644 --- a/pkg/apply/applyuser.go +++ b/pkg/apply/applyuser.go @@ -92,23 +92,9 @@ func (u *UserApplier) Apply(ctx context.Context) error { return fmt.Errorf("error checking existing ACLs for user %s: %v", u.userConfig.Meta.Name, err) } + // TODO: pretty print these ACLs log.Info("Found ", len(existingACLs), " existing ACLs: ", existingACLs) - existingACLEntries := []kafka.ACLEntry{} - for _, acl := range existingACLs { - existingACLEntries = append(existingACLEntries, kafka.ACLEntry{ - ResourceType: kafka.ResourceType(acl.ResourceType), - ResourceName: acl.ResourceName, - ResourcePatternType: kafka.PatternType(acl.PatternType), - Principal: acl.Principal, - Host: acl.Host, - Operation: kafka.ACLOperationType(acl.Operation), - PermissionType: kafka.ACLPermissionType(acl.PermissionType), - }) - } - - // TODO: find orphaned ACLs, new ACLs, existing ACLs - acls := u.userConfig.ToNewACLEntries() // Compare acls and existingACLEntries @@ -118,14 +104,16 @@ func (u *UserApplier) Apply(ctx context.Context) error { } if u.config.DryRun { - log.Infof("Would create ACLs with config %+v", acls) + log.Infof( + "Would create ACLs with config %+v", + FormatNewACLsConfig(acls), + ) return nil } log.Infof( "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", - // TODO: pretty format these ACLs - acls, + FormatNewACLsConfig(acls), ) ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) diff --git a/pkg/apply/format.go b/pkg/apply/format.go index 8c3725ce..40a34e35 100644 --- a/pkg/apply/format.go +++ b/pkg/apply/format.go @@ -37,6 +37,18 @@ func FormatNewUserConfig(config kafka.UserScramCredentialsUpsertion) string { return string(content) } +// FormatNewACLsConfig generates a pretty string representation of kafka-go +// ACL configurations. +func FormatNewACLsConfig(config []kafka.ACLEntry) string { + content, err := json.MarshalIndent(config, "", " ") + if err != nil { + log.Warnf("Error marshalling ACLs config: %+v", err) + return "Error" + } + + return string(content) +} + // FormatSettingsDiff generates a table that summarizes the differences between // the topic settings from a topic config and the settings from ZK. func FormatSettingsDiff( diff --git a/pkg/config/user.go b/pkg/config/user.go index 888feb8a..406af826 100644 --- a/pkg/config/user.go +++ b/pkg/config/user.go @@ -214,3 +214,22 @@ func (u UserConfig) ToNewACLEntries() []kafka.ACLEntry { } return acls } + +func KafkaGoACLEntriesToACLs(acls []kafka.ACLEntry) []ACL { + ACLs := []ACL{} + + for _, acl := range acls { + ACLs = append(ACLs, ACL{ + Resource: ACLResource{ + Type: acl.ResourceType.String(), + Name: acl.ResourceName, + PatternType: acl.ResourcePatternType.String(), + Principal: acl.Principal, + Host: acl.Host, + }, + Operations: []string{acl.Operation.String()}, + }) + } + + return ACLs +} From 107751e6be5636ea650d314d537e57fcee0cb62d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 9 Nov 2023 09:57:51 -0500 Subject: [PATCH 072/116] wip --- cmd/topicctl/subcmd/applyuser.go | 338 +++++++++--------- pkg/apply/applyuser.go | 327 +++++++++--------- pkg/apply/format.go | 12 - pkg/cli/cli.go | 23 +- pkg/config/load.go | 26 +- pkg/config/load_test.go | 91 +++-- pkg/config/user.go | 466 ++++++++++++------------- pkg/config/user_test.go | 572 +++++++++++++++---------------- 8 files changed, 912 insertions(+), 943 deletions(-) diff --git a/cmd/topicctl/subcmd/applyuser.go b/cmd/topicctl/subcmd/applyuser.go index ca00f8be..c7457532 100644 --- a/cmd/topicctl/subcmd/applyuser.go +++ b/cmd/topicctl/subcmd/applyuser.go @@ -1,171 +1,171 @@ package subcmd -import ( - "context" - "fmt" - "os" - "os/signal" - "path/filepath" - "syscall" - - "github.com/segmentio/topicctl/pkg/admin" - "github.com/segmentio/topicctl/pkg/apply" - "github.com/segmentio/topicctl/pkg/cli" - "github.com/segmentio/topicctl/pkg/config" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var applyUserCmd = &cobra.Command{ - Use: "apply-user [user configs]", - Short: "apply one or more user configs", - Args: cobra.MinimumNArgs(1), - RunE: applyUserRun, -} - -func init() { - applyUserCmd.Flags().BoolVar( - &applyConfig.dryRun, - "dry-run", - false, - "Do a dry-run", - ) - applyUserCmd.Flags().StringVar( - &applyConfig.pathPrefix, - "path-prefix", - os.Getenv("TOPICCTL_USER_APPLY_PATH_PREFIX"), - "Prefix for user config paths", - ) - applyUserCmd.Flags().BoolVar( - &applyConfig.skipConfirm, - "skip-confirm", - false, - "Skip confirmation prompts during apply process", - ) - - addSharedConfigOnlyFlags(applyUserCmd, &applyConfig.shared) - RootCmd.AddCommand(applyUserCmd) -} - -func applyUserRun(cmd *cobra.Command, args []string) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - go func() { - <-sigChan - cancel() - }() - - // Keep a cache of the admin clients with the cluster config path as the key - adminClients := map[string]admin.Client{} - - defer func() { - for _, adminClient := range adminClients { - adminClient.Close() - } - }() - - matchCount := 0 - - for _, arg := range args { - if applyConfig.pathPrefix != "" && !filepath.IsAbs(arg) { - arg = filepath.Join(applyConfig.pathPrefix, arg) - } - - matches, err := filepath.Glob(arg) - if err != nil { - return err - } - - for _, match := range matches { - matchCount++ - if err := applyUser(ctx, match, adminClients); err != nil { - return err - } - } - } - - if matchCount == 0 { - return fmt.Errorf("No user configs match the provided args (%+v)", args) - } - - return nil -} - -func applyUser( - ctx context.Context, - userConfigPath string, - adminClients map[string]admin.Client, -) error { - clusterConfigPath, err := clusterConfigForUserApply(userConfigPath) - if err != nil { - return err - } - - userConfigs, err := config.LoadUsersFile(userConfigPath) - if err != nil { - return err - } - - clusterConfig, err := config.LoadClusterFile(clusterConfigPath, applyConfig.shared.expandEnv) - if err != nil { - return err - } - - adminClient, ok := adminClients[clusterConfigPath] - if !ok { - adminClient, err = clusterConfig.NewAdminClient( - ctx, - nil, - applyConfig.dryRun, - applyConfig.shared.saslUsername, - applyConfig.shared.saslPassword, - ) - if err != nil { - return err - } - adminClients[clusterConfigPath] = adminClient - } - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, false) - - for _, userConfig := range userConfigs { - // userConfig.SetDefaults() - log.Infof( - "Processing user %s in config %s with cluster config %s", - userConfig.Meta.Name, - userConfigPath, - clusterConfigPath, - ) - - applierConfig := apply.UserApplierConfig{ - DryRun: applyConfig.dryRun, - SkipConfirm: applyConfig.skipConfirm, - UserConfig: userConfig, - ClusterConfig: clusterConfig, - } - - if err := cliRunner.ApplyUser(ctx, applierConfig); err != nil { - return err - } - } - - return nil -} - -// TODO: move this into a util function shared between this and apply topic -func clusterConfigForUserApply(userConfigPath string) (string, error) { - if applyConfig.shared.clusterConfig != "" { - return applyConfig.shared.clusterConfig, nil - } - - return filepath.Abs( - filepath.Join( - filepath.Dir(userConfigPath), - "..", - "cluster.yaml", - ), - ) -} +// import ( +// "context" +// "fmt" +// "os" +// "os/signal" +// "path/filepath" +// "syscall" + +// "github.com/segmentio/topicctl/pkg/admin" +// "github.com/segmentio/topicctl/pkg/apply" +// "github.com/segmentio/topicctl/pkg/cli" +// "github.com/segmentio/topicctl/pkg/config" +// log "github.com/sirupsen/logrus" +// "github.com/spf13/cobra" +// ) + +// var applyUserCmd = &cobra.Command{ +// Use: "apply-user [user configs]", +// Short: "apply one or more user configs", +// Args: cobra.MinimumNArgs(1), +// RunE: applyUserRun, +// } + +// func init() { +// applyUserCmd.Flags().BoolVar( +// &applyConfig.dryRun, +// "dry-run", +// false, +// "Do a dry-run", +// ) +// applyUserCmd.Flags().StringVar( +// &applyConfig.pathPrefix, +// "path-prefix", +// os.Getenv("TOPICCTL_USER_APPLY_PATH_PREFIX"), +// "Prefix for user config paths", +// ) +// applyUserCmd.Flags().BoolVar( +// &applyConfig.skipConfirm, +// "skip-confirm", +// false, +// "Skip confirmation prompts during apply process", +// ) + +// addSharedConfigOnlyFlags(applyUserCmd, &applyConfig.shared) +// RootCmd.AddCommand(applyUserCmd) +// } + +// func applyUserRun(cmd *cobra.Command, args []string) error { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() + +// sigChan := make(chan os.Signal, 1) +// signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) +// go func() { +// <-sigChan +// cancel() +// }() + +// // Keep a cache of the admin clients with the cluster config path as the key +// adminClients := map[string]admin.Client{} + +// defer func() { +// for _, adminClient := range adminClients { +// adminClient.Close() +// } +// }() + +// matchCount := 0 + +// for _, arg := range args { +// if applyConfig.pathPrefix != "" && !filepath.IsAbs(arg) { +// arg = filepath.Join(applyConfig.pathPrefix, arg) +// } + +// matches, err := filepath.Glob(arg) +// if err != nil { +// return err +// } + +// for _, match := range matches { +// matchCount++ +// if err := applyUser(ctx, match, adminClients); err != nil { +// return err +// } +// } +// } + +// if matchCount == 0 { +// return fmt.Errorf("No user configs match the provided args (%+v)", args) +// } + +// return nil +// } + +// func applyUser( +// ctx context.Context, +// userConfigPath string, +// adminClients map[string]admin.Client, +// ) error { +// clusterConfigPath, err := clusterConfigForUserApply(userConfigPath) +// if err != nil { +// return err +// } + +// userConfigs, err := config.LoadUsersFile(userConfigPath) +// if err != nil { +// return err +// } + +// clusterConfig, err := config.LoadClusterFile(clusterConfigPath, applyConfig.shared.expandEnv) +// if err != nil { +// return err +// } + +// adminClient, ok := adminClients[clusterConfigPath] +// if !ok { +// adminClient, err = clusterConfig.NewAdminClient( +// ctx, +// nil, +// applyConfig.dryRun, +// applyConfig.shared.saslUsername, +// applyConfig.shared.saslPassword, +// ) +// if err != nil { +// return err +// } +// adminClients[clusterConfigPath] = adminClient +// } + +// cliRunner := cli.NewCLIRunner(adminClient, log.Infof, false) + +// for _, userConfig := range userConfigs { +// // userConfig.SetDefaults() +// log.Infof( +// "Processing user %s in config %s with cluster config %s", +// userConfig.Meta.Name, +// userConfigPath, +// clusterConfigPath, +// ) + +// applierConfig := apply.UserApplierConfig{ +// DryRun: applyConfig.dryRun, +// SkipConfirm: applyConfig.skipConfirm, +// UserConfig: userConfig, +// ClusterConfig: clusterConfig, +// } + +// if err := cliRunner.ApplyUser(ctx, applierConfig); err != nil { +// return err +// } +// } + +// return nil +// } + +// // TODO: move this into a util function shared between this and apply topic +// func clusterConfigForUserApply(userConfigPath string) (string, error) { +// if applyConfig.shared.clusterConfig != "" { +// return applyConfig.shared.clusterConfig, nil +// } + +// return filepath.Abs( +// filepath.Join( +// filepath.Dir(userConfigPath), +// "..", +// "cluster.yaml", +// ), +// ) +// } diff --git a/pkg/apply/applyuser.go b/pkg/apply/applyuser.go index e3b99df4..7e3a44b5 100644 --- a/pkg/apply/applyuser.go +++ b/pkg/apply/applyuser.go @@ -1,171 +1,160 @@ package apply -import ( - "context" - "errors" - "fmt" - - "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/admin" - "github.com/segmentio/topicctl/pkg/config" - log "github.com/sirupsen/logrus" -) - -// TODO: dry this up with the apply.go file - -// TODO: are these structs even necessary? -type UserApplierConfig struct { - DryRun bool - SkipConfirm bool - UserConfig config.UserConfig - ClusterConfig config.ClusterConfig -} - -type UserApplier struct { - config UserApplierConfig - adminClient admin.Client - - clusterConfig config.ClusterConfig - userConfig config.UserConfig -} - -func NewUserApplier( - ctx context.Context, - adminClient admin.Client, - applierConfig UserApplierConfig, -) (*UserApplier, error) { - if !adminClient.GetSupportedFeatures().Applies { - return nil, - errors.New( - "Admin client does not support features needed for apply; You need to upgrade to Kafka version >2.7.0.", - ) - } - - return &UserApplier{ - config: applierConfig, - adminClient: adminClient, - clusterConfig: applierConfig.ClusterConfig, - userConfig: applierConfig.UserConfig, - }, nil -} - -func (u *UserApplier) Apply(ctx context.Context) error { - log.Info("Validating configs...") - - if err := u.clusterConfig.Validate(); err != nil { - return fmt.Errorf("error validating cluster config: %v", err) - } - - if err := u.userConfig.Validate(); err != nil { - return fmt.Errorf("error validating user config: %v", err) - } - - log.Info("Checking if user already exists...") - - userInfo, err := u.adminClient.GetUsers(ctx, []string{u.userConfig.Meta.Name}) - if err != nil { - return fmt.Errorf("error checking if user already exists: %v", err) - } - - if len(userInfo) == 0 { - err = u.applyNewUser(ctx) - } else { - // TODO: handle case where this returns multiple users due to multiple creds being created - err = u.applyExistingUser(ctx, userInfo[0]) - } - - if err != nil { - return fmt.Errorf("error applying existing user: %v", err) - } - - log.Info("Checking if ACLs already exist for this user...") - - existingACLs, err := u.adminClient.GetACLs(ctx, kafka.ACLFilter{ - ResourceTypeFilter: kafka.ResourceTypeAny, - ResourcePatternTypeFilter: kafka.PatternTypeAny, - PrincipalFilter: fmt.Sprintf("User:%s", u.userConfig.Meta.Name), - Operation: kafka.ACLOperationTypeAny, - PermissionType: kafka.ACLPermissionTypeAny, - }) - - if err != nil { - return fmt.Errorf("error checking existing ACLs for user %s: %v", u.userConfig.Meta.Name, err) - } - - // TODO: pretty print these ACLs - log.Info("Found ", len(existingACLs), " existing ACLs: ", existingACLs) - - acls := u.userConfig.ToNewACLEntries() - - // Compare acls and existingACLEntries - - if len(acls) == 0 { - return nil - } - - if u.config.DryRun { - log.Infof( - "Would create ACLs with config %+v", - FormatNewACLsConfig(acls), - ) - return nil - } - - log.Infof( - "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", - FormatNewACLsConfig(acls), - ) - - ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) - if !ok { - return errors.New("Stopping because of user response") - } - - log.Infof("Creating new ACLs for user with config %+v", acls) - - err = u.adminClient.CreateACLs(ctx, acls) - if err != nil { - return fmt.Errorf("error creating new ACLs: %v", err) - } - return nil -} - -func (u *UserApplier) applyNewUser(ctx context.Context) error { - user, err := u.userConfig.ToNewUserScramCredentialsUpsertion() - if err != nil { - return fmt.Errorf("error creating UserScramCredentialsUpsertion: %v", err) - } - - if u.config.DryRun { - log.Infof("Would create user with config %+v", user) - return nil - } - - log.Infof( - "It looks like this user doesn't already exists. Will create it with this config:\n%s", - FormatNewUserConfig(user), - ) - - ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) - if !ok { - return errors.New("Stopping because of user response") - } - - log.Infof("Creating new user with config %+v", user) - - err = u.adminClient.UpsertUser(ctx, user) - if err != nil { - return fmt.Errorf("error creating new user: %v", err) - } - - return nil -} - -// TODO: support this -func (u *UserApplier) applyExistingUser( - ctx context.Context, - userInfo admin.UserInfo, -) error { - log.Infof("Updating existing user: %s", u.userConfig.Meta.Name) - return nil -} +// // TODO: dry this up with the apply.go file + +// // TODO: are these structs even necessary? +// type UserApplierConfig struct { +// DryRun bool +// SkipConfirm bool +// UserConfig config.UserConfig +// ClusterConfig config.ClusterConfig +// } + +// type UserApplier struct { +// config UserApplierConfig +// adminClient admin.Client + +// clusterConfig config.ClusterConfig +// userConfig config.UserConfig +// } + +// func NewUserApplier( +// ctx context.Context, +// adminClient admin.Client, +// applierConfig UserApplierConfig, +// ) (*UserApplier, error) { +// if !adminClient.GetSupportedFeatures().Applies { +// return nil, +// errors.New( +// "Admin client does not support features needed for apply; You need to upgrade to Kafka version >2.7.0.", +// ) +// } + +// return &UserApplier{ +// config: applierConfig, +// adminClient: adminClient, +// clusterConfig: applierConfig.ClusterConfig, +// userConfig: applierConfig.UserConfig, +// }, nil +// } + +// func (u *UserApplier) Apply(ctx context.Context) error { +// log.Info("Validating configs...") + +// if err := u.clusterConfig.Validate(); err != nil { +// return fmt.Errorf("error validating cluster config: %v", err) +// } + +// if err := u.userConfig.Validate(); err != nil { +// return fmt.Errorf("error validating user config: %v", err) +// } + +// log.Info("Checking if user already exists...") + +// userInfo, err := u.adminClient.GetUsers(ctx, []string{u.userConfig.Meta.Name}) +// if err != nil { +// return fmt.Errorf("error checking if user already exists: %v", err) +// } + +// if len(userInfo) == 0 { +// err = u.applyNewUser(ctx) +// } else { +// // TODO: handle case where this returns multiple users due to multiple creds being created +// err = u.applyExistingUser(ctx, userInfo[0]) +// } + +// if err != nil { +// return fmt.Errorf("error applying existing user: %v", err) +// } + +// log.Info("Checking if ACLs already exist for this user...") + +// existingACLs, err := u.adminClient.GetACLs(ctx, kafka.ACLFilter{ +// ResourceTypeFilter: kafka.ResourceTypeAny, +// ResourcePatternTypeFilter: kafka.PatternTypeAny, +// PrincipalFilter: fmt.Sprintf("User:%s", u.userConfig.Meta.Name), +// Operation: kafka.ACLOperationTypeAny, +// PermissionType: kafka.ACLPermissionTypeAny, +// }) + +// if err != nil { +// return fmt.Errorf("error checking existing ACLs for user %s: %v", u.userConfig.Meta.Name, err) +// } + +// // TODO: pretty print these ACLs +// log.Info("Found ", len(existingACLs), " existing ACLs: ", existingACLs) + +// acls := u.userConfig.ToNewACLEntries() + +// // Compare acls and existingACLEntries + +// if len(acls) == 0 { +// return nil +// } + +// if u.config.DryRun { +// log.Infof( +// "Would create ACLs with config %+v", +// FormatNewACLsConfig(acls), +// ) +// return nil +// } + +// log.Infof( +// "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", +// FormatNewACLsConfig(acls), +// ) + +// ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) +// if !ok { +// return errors.New("Stopping because of user response") +// } + +// log.Infof("Creating new ACLs for user with config %+v", acls) + +// err = u.adminClient.CreateACLs(ctx, acls) +// if err != nil { +// return fmt.Errorf("error creating new ACLs: %v", err) +// } +// return nil +// } + +// func (u *UserApplier) applyNewUser(ctx context.Context) error { +// user, err := u.userConfig.ToNewUserScramCredentialsUpsertion() +// if err != nil { +// return fmt.Errorf("error creating UserScramCredentialsUpsertion: %v", err) +// } + +// if u.config.DryRun { +// log.Infof("Would create user with config %+v", user) +// return nil +// } + +// log.Infof( +// "It looks like this user doesn't already exists. Will create it with this config:\n%s", +// FormatNewUserConfig(user), +// ) + +// ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) +// if !ok { +// return errors.New("Stopping because of user response") +// } + +// log.Infof("Creating new user with config %+v", user) + +// err = u.adminClient.UpsertUser(ctx, user) +// if err != nil { +// return fmt.Errorf("error creating new user: %v", err) +// } + +// return nil +// } + +// // TODO: support this +// func (u *UserApplier) applyExistingUser( +// ctx context.Context, +// userInfo admin.UserInfo, +// ) error { +// log.Infof("Updating existing user: %s", u.userConfig.Meta.Name) +// return nil +// } diff --git a/pkg/apply/format.go b/pkg/apply/format.go index 40a34e35..8c3725ce 100644 --- a/pkg/apply/format.go +++ b/pkg/apply/format.go @@ -37,18 +37,6 @@ func FormatNewUserConfig(config kafka.UserScramCredentialsUpsertion) string { return string(content) } -// FormatNewACLsConfig generates a pretty string representation of kafka-go -// ACL configurations. -func FormatNewACLsConfig(config []kafka.ACLEntry) string { - content, err := json.MarshalIndent(config, "", " ") - if err != nil { - log.Warnf("Error marshalling ACLs config: %+v", err) - return "Error" - } - - return string(content) -} - // FormatSettingsDiff generates a table that summarizes the differences between // the topic settings from a topic config and the settings from ZK. func FormatSettingsDiff( diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 78ff5564..794ba0bc 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -19,6 +19,7 @@ import ( "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/check" "github.com/segmentio/topicctl/pkg/config" + "github.com/segmentio/topicctl/pkg/create" "github.com/segmentio/topicctl/pkg/groups" "github.com/segmentio/topicctl/pkg/messages" log "github.com/sirupsen/logrus" @@ -111,15 +112,15 @@ func (c *CLIRunner) ApplyTopic( return nil } -// ApplyUser does an apply run according to the spec in the argument config. -func (c *CLIRunner) ApplyUser( +// CreateACL does an apply run according to the spec in the argument config. +func (c *CLIRunner) CreateACL( ctx context.Context, - applierConfig apply.UserApplierConfig, + creatorConfig create.ACLCreatorConfig, ) error { - applier, err := apply.NewUserApplier( + creator, err := create.NewACLCreator( ctx, c.adminClient, - applierConfig, + creatorConfig, ) if err != nil { return err @@ -128,18 +129,18 @@ func (c *CLIRunner) ApplyUser( highlighter := color.New(color.FgYellow, color.Bold).SprintfFunc() c.printer( - "Starting apply for user %s in environment %s, cluster %s", - highlighter(applierConfig.UserConfig.Meta.Name), - highlighter(applierConfig.UserConfig.Meta.Environment), - highlighter(applierConfig.UserConfig.Meta.Cluster), + "Starting creation for ACLs %s in environment %s, cluster %s", + highlighter(creatorConfig.ACLConfig.Meta.Name), + highlighter(creatorConfig.ACLConfig.Meta.Environment), + highlighter(creatorConfig.ACLConfig.Meta.Cluster), ) - err = applier.Apply(ctx) + err = creator.Create(ctx) if err != nil { return err } - c.printer("Apply completed successfully!") + c.printer("Create completed successfully!") return nil } diff --git a/pkg/config/load.go b/pkg/config/load.go index 3ec65ae1..f677cd7f 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -88,8 +88,8 @@ func LoadTopicBytes(contents []byte) (TopicConfig, error) { return config, err } -// LoadUsersFile loads one or more UserConfigs from a path to a YAML file. -func LoadUsersFile(path string) ([]UserConfig, error) { +// LoadACLsFile loads one or more ACLConfigs from a path to a YAML file. +func LoadACLsFile(path string) ([]ACLConfig, error) { contents, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -98,30 +98,30 @@ func LoadUsersFile(path string) ([]UserConfig, error) { contents = []byte(os.ExpandEnv(string(contents))) trimmedFile := strings.TrimSpace(string(contents)) - userStrs := sep.Split(trimmedFile, -1) + aclStrs := sep.Split(trimmedFile, -1) - userConfigs := []UserConfig{} + aclConfigs := []ACLConfig{} - for _, userStr := range userStrs { - userStr = strings.TrimSpace(userStr) - if isEmpty(userStr) { + for _, aclStr := range aclStrs { + aclStr = strings.TrimSpace(aclStr) + if isEmpty(aclStr) { continue } - userConfig, err := LoadUserBytes([]byte(userStr)) + aclConfig, err := LoadACLBytes([]byte(aclStr)) if err != nil { return nil, err } - userConfigs = append(userConfigs, userConfig) + aclConfigs = append(aclConfigs, aclConfig) } - return userConfigs, nil + return aclConfigs, nil } -// LoadUserBytes loads a UserConfig from YAML bytes. -func LoadUserBytes(contents []byte) (UserConfig, error) { - config := UserConfig{} +// LoadACLBytes loads an ACLConfig from YAML bytes. +func LoadACLBytes(contents []byte) (ACLConfig, error) { + config := ACLConfig{} err := unmarshalYAMLStrict(contents, &config) return config, err } diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index f5a599bc..8eb3e229 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "github.com/segmentio/kafka-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -105,72 +106,62 @@ func TestLoadTopicsFile(t *testing.T) { assert.Equal(t, "topic-test2", topicConfigs[1].Meta.Name) } -func TestLoadUsersFile(t *testing.T) { - userConfigs, err := LoadUsersFile("testdata/test-cluster/users/user-test.yaml") +// TODO: write this test +func TestLoadACLsFile(t *testing.T) { + aclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test.yaml") require.NoError(t, err) - fmt.Println(userConfigs) - assert.Equal(t, 1, len(userConfigs)) - userConfig := userConfigs[0] + fmt.Println(aclConfigs) + assert.Equal(t, 1, len(aclConfigs)) + aclConfig := aclConfigs[0] assert.Equal( t, - UserConfig{ - Meta: UserMeta{ - Name: "user-test", + ACLConfig{ + Meta: ACLMeta{ + Name: "acl-test", Cluster: "test-cluster", Region: "test-region", Environment: "test-env", - Description: "Test user\n", + Description: "Test acl\n", }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - }, - Operations: []string{ - "read", - "describe", - }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: "test-topic", + PatternType: kafka.PatternTypeLiteral, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + kafka.ACLOperationTypeDescribe, + }, + }, + { + Resource: ACLResource{ + Type: kafka.ResourceTypeGroup, + Name: "test-group", + PatternType: kafka.PatternTypePrefixed, }, - { - Resource: ACLResource{ - Type: "group", - Name: "test-group", - PatternType: "prefixed", - }, - Operations: []string{ - "read", - }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, }, }, }, }, }, - userConfig, + aclConfig, ) - assert.NoError(t, userConfig.Validate()) - invalidUserConfigs, err := LoadUsersFile("testdata/test-cluster/users/user-test-invalid.yaml") - assert.Equal(t, 1, len(invalidUserConfigs)) - invalidUserConfig := invalidUserConfigs[0] - require.NoError(t, err) - assert.Error(t, invalidUserConfig.Validate()) - - multiUserConfigs, err := LoadUsersFile("testdata/test-cluster/users/user-test-multi.yaml") - assert.Equal(t, 2, len(multiUserConfigs)) - assert.Equal(t, "user-test1", multiUserConfigs[0].Meta.Name) - assert.Equal(t, "user-test2", multiUserConfigs[1].Meta.Name) - assert.NoError(t, multiUserConfigs[0].Validate()) - assert.NoError(t, multiUserConfigs[1].Validate()) + invalidAclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test-invalid.yaml") + assert.Equal(t, 0, len(invalidAclConfigs)) + // TODO: improve this error checking and make sure the error is informative enough + require.Error(t, err) + + multiAclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test-multi.yaml") + assert.Equal(t, 2, len(multiAclConfigs)) + assert.Equal(t, "acl-test1", multiAclConfigs[0].Meta.Name) + assert.Equal(t, "acl-test2", multiAclConfigs[1].Meta.Name) } func TestCheckConsistency(t *testing.T) { diff --git a/pkg/config/user.go b/pkg/config/user.go index 406af826..11785474 100644 --- a/pkg/config/user.go +++ b/pkg/config/user.go @@ -1,235 +1,235 @@ package config -import ( - "crypto/rand" - "crypto/sha512" - "errors" - "fmt" - - "github.com/hashicorp/go-multierror" - "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/admin" - "github.com/xdg-go/pbkdf2" -) - -type UserConfig struct { - Meta UserMeta `json:"meta"` - Spec UserSpec `json:"spec"` -} - -type UserMeta struct { - Name string `json:"name"` - Cluster string `json:"cluster"` - Region string `json:"region"` - Environment string `json:"environment"` - Description string `json:"description"` - Labels map[string]string `json:"labels"` -} - -type UserSpec struct { - Authentication AuthenticationConfig `json:"authentication,omitempty"` - Authorization AuthorizationConfig `json:"authorization,omitempty"` -} - -type AuthenticationConfig struct { - Type AuthenticationType `json:"type"` - // TODO: extend this to a type that supports SSMRef - Password string `json:"password"` -} - -type AuthenticationType string - -const ( - ScramSha512 AuthenticationType = "scram-sha-512" -) - -var allAuthenticationTypes = []AuthenticationType{ - ScramSha512, -} - -type AuthorizationConfig struct { - Type AuthorizationType `json:"type"` - ACLs []ACL `json:"acls,omitempty"` -} - -type AuthorizationType string - -const ( - SimpleAuthorization AuthorizationType = "simple" -) - -var allAuthorizationTypes = []AuthorizationType{ - SimpleAuthorization, -} - -type ACL struct { - Resource ACLResource `json:"resource"` - Operations []string `json:"operations"` -} - -// TODO: how should principal and permission type be handled? -// principal will always be the meta name and permission type will always be allowed -type ACLResource struct { - Type string `json:"type"` - Name string `json:"name"` - PatternType string `json:"patternType"` - Principal string `json:"principal"` - Host string `json:"host"` -} - -func keys[K comparable, V any](m map[K]V) []K { - keys := make([]K, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - return keys -} - -var allResourceTypes = keys(admin.ResourceTypeMap) -var allPatternTypes = keys(admin.PatternTypeMap) -var allOperationTypes = keys(admin.AclOperationTypeMap) - -func (u *UserConfig) SetDefaults() { - if u.Spec.Authorization.Type == "" { - u.Spec.Authorization.Type = SimpleAuthorization - } -} - -func (u *UserConfig) Validate() error { - // TODO: validate password types - var err error - - if u.Meta.Name == "" { - err = multierror.Append(err, errors.New("Name must be set")) - } - if u.Meta.Cluster == "" { - err = multierror.Append(err, errors.New("Cluster must be set")) - } - if u.Meta.Region == "" { - err = multierror.Append(err, errors.New("Region must be set")) - } - if u.Meta.Environment == "" { - err = multierror.Append(err, errors.New("Environment must be set")) - } - - authenticationTypeFound := false - for _, authenticationType := range allAuthenticationTypes { - if authenticationType == u.Spec.Authentication.Type { - authenticationTypeFound = true - } - } - - if !authenticationTypeFound { - err = multierror.Append( - err, - fmt.Errorf("Authentication Type must be in %+v, got: %s", allAuthenticationTypes, u.Spec.Authentication.Type), - ) - } - - authorizationTypeFound := false - for _, authorizationType := range allAuthorizationTypes { - if authorizationType == u.Spec.Authorization.Type { - authorizationTypeFound = true - } - } - - if !authorizationTypeFound { - err = multierror.Append( - err, - fmt.Errorf("Authorization Type must be in %+v, got: %s", allAuthorizationTypes, u.Spec.Authorization.Type), - ) - } - - for _, acl := range u.Spec.Authorization.ACLs { - if _, ok := admin.ResourceTypeMap[acl.Resource.Type]; !ok { - err = multierror.Append( - err, - fmt.Errorf("ACL Resource Type must be in %+v, got: %s", allResourceTypes, acl.Resource.Type), - ) - } - if _, ok := admin.PatternTypeMap[acl.Resource.PatternType]; !ok { - err = multierror.Append( - err, - fmt.Errorf("ACL Resource PatternType must be in %+v, got: %s", allPatternTypes, acl.Resource.PatternType), - ) - } - for _, operation := range acl.Operations { - if _, ok := admin.AclOperationTypeMap[operation]; !ok { - err = multierror.Append( - err, - fmt.Errorf("ACL OperationType must be in %+v, got: %s", allOperationTypes, operation), - ) - } - } - } - - return err -} - -const ( - // Currently only scram-sha-512 is supported - ScramMechanism kafka.ScramMechanism = kafka.ScramMechanismSha512 - // Use the same default as Postgres and Strimzi for Scram iterations - ScramIterations int = 4096 -) - -func (u UserConfig) ToNewUserScramCredentialsUpsertion() (kafka.UserScramCredentialsUpsertion, error) { - salt := make([]byte, 24) - if _, err := rand.Read(salt); err != nil { - return kafka.UserScramCredentialsUpsertion{}, fmt.Errorf("User %s: unable to generate salt: %v", u.Meta.Name, err) - } - saltedPassword := pbkdf2.Key([]byte(u.Spec.Authentication.Password), salt, ScramIterations, sha512.Size, sha512.New) - - return kafka.UserScramCredentialsUpsertion{ - Name: u.Meta.Name, - Mechanism: ScramMechanism, - Iterations: ScramIterations, - Salt: salt, - SaltedPassword: saltedPassword, - }, nil -} - -func (u UserConfig) ToNewACLEntries() []kafka.ACLEntry { - acls := []kafka.ACLEntry{} - - for _, acl := range u.Spec.Authorization.ACLs { - // Data has already been validated before calling this function so no need to check validity - - resourceType, _ := admin.ResourceTypeMap[acl.Resource.Type] - resourcePatternType, _ := admin.PatternTypeMap[acl.Resource.PatternType] - - for _, operation := range acl.Operations { - aclOperation, _ := admin.AclOperationTypeMap[operation] - - acls = append(acls, kafka.ACLEntry{ - ResourceType: resourceType, - ResourceName: acl.Resource.Name, - ResourcePatternType: resourcePatternType, - Principal: fmt.Sprintf("User:%s", u.Meta.Name), - Host: acl.Resource.Host, - Operation: aclOperation, - PermissionType: kafka.ACLPermissionTypeAllow, - }) - } - } - return acls -} - -func KafkaGoACLEntriesToACLs(acls []kafka.ACLEntry) []ACL { - ACLs := []ACL{} - - for _, acl := range acls { - ACLs = append(ACLs, ACL{ - Resource: ACLResource{ - Type: acl.ResourceType.String(), - Name: acl.ResourceName, - PatternType: acl.ResourcePatternType.String(), - Principal: acl.Principal, - Host: acl.Host, - }, - Operations: []string{acl.Operation.String()}, - }) - } - - return ACLs -} +// import ( +// "crypto/rand" +// "crypto/sha512" +// "errors" +// "fmt" + +// "github.com/hashicorp/go-multierror" +// "github.com/segmentio/kafka-go" +// "github.com/segmentio/topicctl/pkg/admin" +// "github.com/xdg-go/pbkdf2" +// ) + +// type UserConfig struct { +// Meta UserMeta `json:"meta"` +// Spec UserSpec `json:"spec"` +// } + +// type UserMeta struct { +// Name string `json:"name"` +// Cluster string `json:"cluster"` +// Region string `json:"region"` +// Environment string `json:"environment"` +// Description string `json:"description"` +// Labels map[string]string `json:"labels"` +// } + +// type UserSpec struct { +// Authentication AuthenticationConfig `json:"authentication,omitempty"` +// Authorization AuthorizationConfig `json:"authorization,omitempty"` +// } + +// type AuthenticationConfig struct { +// Type AuthenticationType `json:"type"` +// // TODO: extend this to a type that supports SSMRef +// Password string `json:"password"` +// } + +// type AuthenticationType string + +// const ( +// ScramSha512 AuthenticationType = "scram-sha-512" +// ) + +// var allAuthenticationTypes = []AuthenticationType{ +// ScramSha512, +// } + +// type AuthorizationConfig struct { +// Type AuthorizationType `json:"type"` +// ACLs []ACL `json:"acls,omitempty"` +// } + +// type AuthorizationType string + +// const ( +// SimpleAuthorization AuthorizationType = "simple" +// ) + +// var allAuthorizationTypes = []AuthorizationType{ +// SimpleAuthorization, +// } + +// type ACL struct { +// Resource ACLResource `json:"resource"` +// Operations []string `json:"operations"` +// } + +// // TODO: how should principal and permission type be handled? +// // principal will always be the meta name and permission type will always be allowed +// type ACLResource struct { +// Type string `json:"type"` +// Name string `json:"name"` +// PatternType string `json:"patternType"` +// Principal string `json:"principal"` +// Host string `json:"host"` +// } + +// func keys[K comparable, V any](m map[K]V) []K { +// keys := make([]K, 0, len(m)) +// for k := range m { +// keys = append(keys, k) +// } +// return keys +// } + +// var allResourceTypes = keys(admin.ResourceTypeMap) +// var allPatternTypes = keys(admin.PatternTypeMap) +// var allOperationTypes = keys(admin.AclOperationTypeMap) + +// func (u *UserConfig) SetDefaults() { +// if u.Spec.Authorization.Type == "" { +// u.Spec.Authorization.Type = SimpleAuthorization +// } +// } + +// func (u *UserConfig) Validate() error { +// // TODO: validate password types +// var err error + +// if u.Meta.Name == "" { +// err = multierror.Append(err, errors.New("Name must be set")) +// } +// if u.Meta.Cluster == "" { +// err = multierror.Append(err, errors.New("Cluster must be set")) +// } +// if u.Meta.Region == "" { +// err = multierror.Append(err, errors.New("Region must be set")) +// } +// if u.Meta.Environment == "" { +// err = multierror.Append(err, errors.New("Environment must be set")) +// } + +// authenticationTypeFound := false +// for _, authenticationType := range allAuthenticationTypes { +// if authenticationType == u.Spec.Authentication.Type { +// authenticationTypeFound = true +// } +// } + +// if !authenticationTypeFound { +// err = multierror.Append( +// err, +// fmt.Errorf("Authentication Type must be in %+v, got: %s", allAuthenticationTypes, u.Spec.Authentication.Type), +// ) +// } + +// authorizationTypeFound := false +// for _, authorizationType := range allAuthorizationTypes { +// if authorizationType == u.Spec.Authorization.Type { +// authorizationTypeFound = true +// } +// } + +// if !authorizationTypeFound { +// err = multierror.Append( +// err, +// fmt.Errorf("Authorization Type must be in %+v, got: %s", allAuthorizationTypes, u.Spec.Authorization.Type), +// ) +// } + +// for _, acl := range u.Spec.Authorization.ACLs { +// if _, ok := admin.ResourceTypeMap[acl.Resource.Type]; !ok { +// err = multierror.Append( +// err, +// fmt.Errorf("ACL Resource Type must be in %+v, got: %s", allResourceTypes, acl.Resource.Type), +// ) +// } +// if _, ok := admin.PatternTypeMap[acl.Resource.PatternType]; !ok { +// err = multierror.Append( +// err, +// fmt.Errorf("ACL Resource PatternType must be in %+v, got: %s", allPatternTypes, acl.Resource.PatternType), +// ) +// } +// for _, operation := range acl.Operations { +// if _, ok := admin.AclOperationTypeMap[operation]; !ok { +// err = multierror.Append( +// err, +// fmt.Errorf("ACL OperationType must be in %+v, got: %s", allOperationTypes, operation), +// ) +// } +// } +// } + +// return err +// } + +// const ( +// // Currently only scram-sha-512 is supported +// ScramMechanism kafka.ScramMechanism = kafka.ScramMechanismSha512 +// // Use the same default as Postgres and Strimzi for Scram iterations +// ScramIterations int = 4096 +// ) + +// func (u UserConfig) ToNewUserScramCredentialsUpsertion() (kafka.UserScramCredentialsUpsertion, error) { +// salt := make([]byte, 24) +// if _, err := rand.Read(salt); err != nil { +// return kafka.UserScramCredentialsUpsertion{}, fmt.Errorf("User %s: unable to generate salt: %v", u.Meta.Name, err) +// } +// saltedPassword := pbkdf2.Key([]byte(u.Spec.Authentication.Password), salt, ScramIterations, sha512.Size, sha512.New) + +// return kafka.UserScramCredentialsUpsertion{ +// Name: u.Meta.Name, +// Mechanism: ScramMechanism, +// Iterations: ScramIterations, +// Salt: salt, +// SaltedPassword: saltedPassword, +// }, nil +// } + +// func (u UserConfig) ToNewACLEntries() []kafka.ACLEntry { +// acls := []kafka.ACLEntry{} + +// for _, acl := range u.Spec.Authorization.ACLs { +// // Data has already been validated before calling this function so no need to check validity + +// resourceType, _ := admin.ResourceTypeMap[acl.Resource.Type] +// resourcePatternType, _ := admin.PatternTypeMap[acl.Resource.PatternType] + +// for _, operation := range acl.Operations { +// aclOperation, _ := admin.AclOperationTypeMap[operation] + +// acls = append(acls, kafka.ACLEntry{ +// ResourceType: resourceType, +// ResourceName: acl.Resource.Name, +// ResourcePatternType: resourcePatternType, +// Principal: fmt.Sprintf("User:%s", u.Meta.Name), +// Host: acl.Resource.Host, +// Operation: aclOperation, +// PermissionType: kafka.ACLPermissionTypeAllow, +// }) +// } +// } +// return acls +// } + +// func KafkaGoACLEntriesToACLs(acls []kafka.ACLEntry) []ACL { +// ACLs := []ACL{} + +// for _, acl := range acls { +// ACLs = append(ACLs, ACL{ +// Resource: ACLResource{ +// Type: acl.ResourceType.String(), +// Name: acl.ResourceName, +// PatternType: acl.ResourcePatternType.String(), +// Principal: acl.Principal, +// Host: acl.Host, +// }, +// Operations: []string{acl.Operation.String()}, +// }) +// } + +// return ACLs +// } diff --git a/pkg/config/user_test.go b/pkg/config/user_test.go index 283d45ad..475ee1c3 100644 --- a/pkg/config/user_test.go +++ b/pkg/config/user_test.go @@ -1,294 +1,294 @@ package config -import ( - "testing" +// import ( +// "testing" - "github.com/stretchr/testify/assert" -) +// "github.com/stretchr/testify/assert" +// ) -func TestUserValidate(t *testing.T) { - type testCase struct { - description string - userConfig UserConfig - expError bool - } +// func TestUserValidate(t *testing.T) { +// type testCase struct { +// description string +// userConfig UserConfig +// expError bool +// } - testCases := []testCase{ - { - description: "happy path", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - Description: "Bootstrapped via topicctl bootstrap", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - }, - expError: false, - }, - { - description: "missing meta fields", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - }, - expError: true, - }, - { - description: "invalid authentication type", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - Description: "Bootstrapped via topicctl bootstrap", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "invalid", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - }, - expError: true, - }, - { - description: "invalid authorization type", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - Description: "Bootstrapped via topicctl bootstrap", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "invalid", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - }, - expError: true, - }, - { - description: "invalid ACL resource type", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - Description: "Bootstrapped via topicctl bootstrap", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "invalid", - Name: "test-topic", - PatternType: "literal", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - }, - expError: true, - }, - { - description: "invalid ACL resource pattern type", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - Description: "Bootstrapped via topicctl bootstrap", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "invalid", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - }, - expError: true, - }, - { - description: "invalid ACL operation type", - userConfig: UserConfig{ - Meta: UserMeta{ - Name: "test-user", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - Description: "Bootstrapped via topicctl bootstrap", - }, - Spec: UserSpec{ - Authentication: AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: AuthorizationConfig{ - Type: "simple", - ACLs: []ACL{ - { - Resource: ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - Principal: "User:alice", - Host: "*", - }, - Operations: []string{ - "invalid", - "describe", - }, - }, - }, - }, - }, - }, - expError: true, - }, - } +// testCases := []testCase{ +// { +// description: "happy path", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// Environment: "test-environment", +// Description: "Bootstrapped via topicctl bootstrap", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "scram-sha-512", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "simple", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "topic", +// Name: "test-topic", +// PatternType: "literal", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "read", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: false, +// }, +// { +// description: "missing meta fields", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "scram-sha-512", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "simple", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "topic", +// Name: "test-topic", +// PatternType: "literal", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "read", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: true, +// }, +// { +// description: "invalid authentication type", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// Environment: "test-environment", +// Description: "Bootstrapped via topicctl bootstrap", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "invalid", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "simple", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "topic", +// Name: "test-topic", +// PatternType: "literal", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "read", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: true, +// }, +// { +// description: "invalid authorization type", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// Environment: "test-environment", +// Description: "Bootstrapped via topicctl bootstrap", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "scram-sha-512", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "invalid", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "topic", +// Name: "test-topic", +// PatternType: "literal", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "read", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: true, +// }, +// { +// description: "invalid ACL resource type", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// Environment: "test-environment", +// Description: "Bootstrapped via topicctl bootstrap", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "scram-sha-512", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "simple", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "invalid", +// Name: "test-topic", +// PatternType: "literal", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "read", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: true, +// }, +// { +// description: "invalid ACL resource pattern type", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// Environment: "test-environment", +// Description: "Bootstrapped via topicctl bootstrap", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "scram-sha-512", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "simple", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "topic", +// Name: "test-topic", +// PatternType: "invalid", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "read", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: true, +// }, +// { +// description: "invalid ACL operation type", +// userConfig: UserConfig{ +// Meta: UserMeta{ +// Name: "test-user", +// Cluster: "test-cluster", +// Region: "test-region", +// Environment: "test-environment", +// Description: "Bootstrapped via topicctl bootstrap", +// }, +// Spec: UserSpec{ +// Authentication: AuthenticationConfig{ +// Type: "scram-sha-512", +// Password: "test-password", +// }, +// Authorization: AuthorizationConfig{ +// Type: "simple", +// ACLs: []ACL{ +// { +// Resource: ACLResource{ +// Type: "topic", +// Name: "test-topic", +// PatternType: "literal", +// Principal: "User:alice", +// Host: "*", +// }, +// Operations: []string{ +// "invalid", +// "describe", +// }, +// }, +// }, +// }, +// }, +// }, +// expError: true, +// }, +// } - for _, testCase := range testCases { - err := testCase.userConfig.Validate() - if testCase.expError { - assert.Error(t, err, testCase.description) - } else { - assert.NoError(t, err, testCase.description) - } - } -} +// for _, testCase := range testCases { +// err := testCase.userConfig.Validate() +// if testCase.expError { +// assert.Error(t, err, testCase.description) +// } else { +// assert.NoError(t, err, testCase.description) +// } +// } +// } -// TODO: write this test -func TestUserConfigFromUserInfo(t *testing.T) { - t.Fatal("implement me") -} +// // TODO: write this test +// func TestUserConfigFromUserInfo(t *testing.T) { +// t.Fatal("implement me") +// } -// TODO: write this test -func TestACLEntryFromUser(t *testing.T) { - t.Fatal("implement me") -} +// // TODO: write this test +// func TestACLEntryFromUser(t *testing.T) { +// t.Fatal("implement me") +// } From aa72764934551b3023e2d86893b54e51c2c37e7e Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 9 Nov 2023 10:27:21 -0500 Subject: [PATCH 073/116] add more files --- cmd/topicctl/subcmd/create.go | 201 ++++++++++++++++++++++++++++++++++ pkg/config/acl.go | 56 ++++++++++ 2 files changed, 257 insertions(+) create mode 100644 cmd/topicctl/subcmd/create.go create mode 100644 pkg/config/acl.go diff --git a/cmd/topicctl/subcmd/create.go b/cmd/topicctl/subcmd/create.go new file mode 100644 index 00000000..15fe66a3 --- /dev/null +++ b/cmd/topicctl/subcmd/create.go @@ -0,0 +1,201 @@ +package subcmd + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/cli" + "github.com/segmentio/topicctl/pkg/config" + "github.com/segmentio/topicctl/pkg/create" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var createCmd = &cobra.Command{ + Use: "create [resource type]", + Short: "creates one or more resources", + PersistentPreRunE: createPreRun, +} + +type createCmdConfig struct { + dryRun bool + pathPrefix string + skipConfirm bool + + shared sharedOptions +} + +var createConfig createCmdConfig + +func init() { + createCmd.Flags().BoolVar( + &createConfig.dryRun, + "dry-run", + false, + "Do a dry-run", + ) + createCmd.Flags().StringVar( + &createConfig.pathPrefix, + "path-prefix", + os.Getenv("TOPICCTL_ACL_PATH_PREFIX"), + "Prefix for ACL config paths", + ) + createCmd.Flags().BoolVar( + &createConfig.skipConfirm, + "skip-confirm", + false, + "Skip confirmation prompts during creation process", + ) + + addSharedFlags(createCmd, &createConfig.shared) + createCmd.AddCommand( + createACLsCmd(), + ) + RootCmd.AddCommand(createCmd) +} + +func createPreRun(cmd *cobra.Command, args []string) error { + if err := RootCmd.PersistentPreRunE(cmd, args); err != nil { + return err + } + return createConfig.shared.validate() +} + +func createACLsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "acls [acl configs]", + Short: "creates ACLs from configuration files", + Args: cobra.MinimumNArgs(1), + RunE: createACLRun, + PreRunE: createPreRun, + } + + return cmd +} + +func createACLRun(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + go func() { + <-sigChan + cancel() + }() + + // Keep a cache of the admin clients with the cluster config path as the key + adminClients := map[string]admin.Client{} + + defer func() { + for _, adminClient := range adminClients { + adminClient.Close() + } + }() + + matchCount := 0 + + for _, arg := range args { + if createConfig.pathPrefix != "" && !filepath.IsAbs(arg) { + arg = filepath.Join(createConfig.pathPrefix, arg) + } + + matches, err := filepath.Glob(arg) + if err != nil { + return err + } + + for _, match := range matches { + matchCount++ + if err := createACL(ctx, match, adminClients); err != nil { + return err + } + } + } + + if matchCount == 0 { + return fmt.Errorf("No ACL configs match the provided args (%+v)", args) + } + + return nil +} + +func createACL( + ctx context.Context, + aclConfigPath string, + adminClients map[string]admin.Client, +) error { + // TODO: check consistency of cluster config and ACL config + clusterConfigPath, err := clusterConfigForACLCreate(aclConfigPath) + if err != nil { + return err + } + + aclConfigs, err := config.LoadACLsFile(aclConfigPath) + if err != nil { + return err + } + + clusterConfig, err := config.LoadClusterFile(clusterConfigPath, createConfig.shared.expandEnv) + if err != nil { + return err + } + + adminClient, ok := adminClients[clusterConfigPath] + if !ok { + adminClient, err = clusterConfig.NewAdminClient( + ctx, + nil, + applyConfig.dryRun, + applyConfig.shared.saslUsername, + applyConfig.shared.saslPassword, + ) + if err != nil { + return err + } + adminClients[clusterConfigPath] = adminClient + } + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, false) + + for _, aclConfig := range aclConfigs { + log.Infof( + "Processing ACL %s in config %s with cluster config %s", + aclConfig.Meta.Name, + aclConfigPath, + clusterConfigPath, + ) + + creatorConfig := create.ACLCreatorConfig{ + DryRun: applyConfig.dryRun, + SkipConfirm: applyConfig.skipConfirm, + ACLConfig: aclConfig, + ClusterConfig: clusterConfig, + } + + if err := cliRunner.CreateACL(ctx, creatorConfig); err != nil { + return err + } + } + + return nil +} + +func clusterConfigForACLCreate(aclConfigPath string) (string, error) { + if createConfig.shared.clusterConfig != "" { + return createConfig.shared.clusterConfig, nil + } + + return filepath.Abs( + filepath.Join( + filepath.Dir(aclConfigPath), + "..", + "cluster.yaml", + ), + ) +} diff --git a/pkg/config/acl.go b/pkg/config/acl.go new file mode 100644 index 00000000..6fb71141 --- /dev/null +++ b/pkg/config/acl.go @@ -0,0 +1,56 @@ +package config + +import ( + "github.com/segmentio/kafka-go" +) + +type ACLConfig struct { + Meta ACLMeta `json:"meta"` + Spec ACLSpec `json:"spec"` +} + +type ACLMeta struct { + Name string `json:"name"` + Cluster string `json:"cluster"` + Region string `json:"region"` + Environment string `json:"environment"` + Description string `json:"description"` + Labels map[string]string `json:"labels"` +} + +type ACLSpec struct { + ACLs []ACL `json:"acls"` +} + +type ACL struct { + Resource ACLResource `json:"resource"` + Operations []kafka.ACLOperationType `json:"operations"` +} + +type ACLResource struct { + Type kafka.ResourceType `json:"type"` + Name string `json:"name"` + PatternType kafka.PatternType `json:"patternType"` + Principal string `json:"principal"` + Host string `json:"host"` + Permission kafka.ACLPermissionType `json:"permission"` +} + +func (a ACLConfig) ToNewACLEntries() []kafka.ACLEntry { + acls := []kafka.ACLEntry{} + + for _, acl := range a.Spec.ACLs { + for _, operation := range acl.Operations { + acls = append(acls, kafka.ACLEntry{ + ResourceType: acl.Resource.Type, + ResourceName: acl.Resource.Name, + ResourcePatternType: acl.Resource.PatternType, + Principal: acl.Resource.Principal, + Host: acl.Resource.Host, + Operation: operation, + PermissionType: acl.Resource.Permission, + }) + } + } + return acls +} From 61e6925bdec1201fec79d49ad4a4081c67a1890f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 15:41:44 -0500 Subject: [PATCH 074/116] resourcemta --- cmd/topicctl/subcmd/rebalance.go | 5 +++-- examples/auth/cluster.yaml | 1 - pkg/apply/apply.go | 2 +- pkg/apply/apply_test.go | 18 +++++++++--------- pkg/check/check.go | 2 +- pkg/check/check_test.go | 8 ++++---- pkg/config/acl.go | 13 ++----------- pkg/config/load.go | 8 ++++---- pkg/config/load_test.go | 6 +++--- pkg/config/topic.go | 21 +++------------------ pkg/config/topic_test.go | 30 +++++++++++++++--------------- 11 files changed, 45 insertions(+), 69 deletions(-) diff --git a/cmd/topicctl/subcmd/rebalance.go b/cmd/topicctl/subcmd/rebalance.go index 7da0096f..a85150d6 100644 --- a/cmd/topicctl/subcmd/rebalance.go +++ b/cmd/topicctl/subcmd/rebalance.go @@ -3,7 +3,6 @@ package subcmd import ( "context" "fmt" - "github.com/spf13/cobra" "os" "os/signal" "path/filepath" @@ -11,6 +10,8 @@ import ( "syscall" "time" + "github.com/spf13/cobra" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/cli" @@ -159,7 +160,7 @@ func rebalanceRun(cmd *cobra.Command, args []string) error { for _, topicConfig := range topicConfigs { // topic config should be consistent with the cluster config - if err := config.CheckConsistency(topicConfig, clusterConfig); err != nil { + if err := config.CheckConsistency(topicConfig.Meta, clusterConfig); err != nil { log.Errorf("topic file: %s inconsistent with cluster: %s", topicFile, clusterConfigPath) continue } diff --git a/examples/auth/cluster.yaml b/examples/auth/cluster.yaml index b6c92123..0972d7fb 100644 --- a/examples/auth/cluster.yaml +++ b/examples/auth/cluster.yaml @@ -19,7 +19,6 @@ spec: skipVerify: true sasl: enabled: true - mechanism: SCRAM-SHA-512 # As an alternative to storing these in plain text in the config (probably not super-secure), # these can also be set via: diff --git a/pkg/apply/apply.go b/pkg/apply/apply.go index 6d55048a..76e54f0f 100644 --- a/pkg/apply/apply.go +++ b/pkg/apply/apply.go @@ -130,7 +130,7 @@ func (t *TopicApplier) Apply(ctx context.Context) error { if err := t.topicConfig.Validate(len(brokerRacks)); err != nil { return err } - if err := config.CheckConsistency(t.topicConfig, t.clusterConfig); err != nil { + if err := config.CheckConsistency(t.topicConfig.Meta, t.clusterConfig); err != nil { return err } diff --git a/pkg/apply/apply_test.go b/pkg/apply/apply_test.go index 5f8e0b88..450beaec 100644 --- a/pkg/apply/apply_test.go +++ b/pkg/apply/apply_test.go @@ -20,7 +20,7 @@ func TestApplyBasicUpdates(t *testing.T) { topicName := util.RandomString("apply-topic-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -88,7 +88,7 @@ func TestApplyPlacementUpdates(t *testing.T) { topicName := util.RandomString("apply-topic-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -206,7 +206,7 @@ func TestApplyRebalance(t *testing.T) { topicName := util.RandomString("apply-topic-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -285,7 +285,7 @@ func TestApplyExtendPartitions(t *testing.T) { topicName := util.RandomString("apply-topic-extend-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -376,7 +376,7 @@ func TestApplyExistingThrottles(t *testing.T) { topicName2 := util.RandomString("apply-topic-extend-", 6) topicConfig1 := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName1, Cluster: "test-cluster", Region: "test-region", @@ -397,7 +397,7 @@ func TestApplyExistingThrottles(t *testing.T) { }, } topicConfig2 := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName2, Cluster: "test-cluster", Region: "test-region", @@ -555,7 +555,7 @@ func TestApplyDryRun(t *testing.T) { topicName := util.RandomString("apply-topic-dry-run-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -620,7 +620,7 @@ func TestApplyThrottles(t *testing.T) { topicName := util.RandomString("apply-topic-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -858,7 +858,7 @@ func TestApplyOverrides(t *testing.T) { topicName := util.RandomString("apply-topic-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", diff --git a/pkg/check/check.go b/pkg/check/check.go index 7b7463c6..70be6098 100644 --- a/pkg/check/check.go +++ b/pkg/check/check.go @@ -48,7 +48,7 @@ func CheckTopic(ctx context.Context, config CheckConfig) (TopicCheckResults, err Name: CheckNameConfigsConsistent, }, ) - if err := tconfig.CheckConsistency(config.TopicConfig, config.ClusterConfig); err == nil { + if err := tconfig.CheckConsistency(config.TopicConfig.Meta, config.ClusterConfig); err == nil { results.UpdateLastResult(true, "") } else { results.UpdateLastResult( diff --git a/pkg/check/check_test.go b/pkg/check/check_test.go index b070d824..455ba135 100644 --- a/pkg/check/check_test.go +++ b/pkg/check/check_test.go @@ -33,7 +33,7 @@ func TestCheck(t *testing.T) { topicName := util.RandomString("check-topic-", 6) topicConfig := config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", @@ -106,7 +106,7 @@ func TestCheck(t *testing.T) { { description: "topic does not exist", checkTopicConfig: config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: "non-existent-topic", Cluster: "non-matching-cluster", Region: "test-region", @@ -134,7 +134,7 @@ func TestCheck(t *testing.T) { { description: "topic does not exist", checkTopicConfig: config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: "non-existent-topic", Cluster: "test-cluster", Region: "test-region", @@ -163,7 +163,7 @@ func TestCheck(t *testing.T) { { description: "wrong configuration", checkTopicConfig: config.TopicConfig{ - Meta: config.TopicMeta{ + Meta: config.ResourceMeta{ Name: topicName, Cluster: "test-cluster", Region: "test-region", diff --git a/pkg/config/acl.go b/pkg/config/acl.go index 6fb71141..189912fe 100644 --- a/pkg/config/acl.go +++ b/pkg/config/acl.go @@ -5,17 +5,8 @@ import ( ) type ACLConfig struct { - Meta ACLMeta `json:"meta"` - Spec ACLSpec `json:"spec"` -} - -type ACLMeta struct { - Name string `json:"name"` - Cluster string `json:"cluster"` - Region string `json:"region"` - Environment string `json:"environment"` - Description string `json:"description"` - Labels map[string]string `json:"labels"` + Meta ResourceMeta `json:"meta"` + Spec ACLSpec `json:"spec"` } type ACLSpec struct { diff --git a/pkg/config/load.go b/pkg/config/load.go index f677cd7f..0ae8f922 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -128,22 +128,22 @@ func LoadACLBytes(contents []byte) (ACLConfig, error) { // CheckConsistency verifies that the argument topic config is consistent with the argument // cluster, e.g. has the same environment and region, etc. -func CheckConsistency(topicConfig TopicConfig, clusterConfig ClusterConfig) error { +func CheckConsistency(resourceMeta ResourceMeta, clusterConfig ClusterConfig) error { var err error - if topicConfig.Meta.Cluster != clusterConfig.Meta.Name { + if resourceMeta.Cluster != clusterConfig.Meta.Name { err = multierror.Append( err, errors.New("Topic cluster name does not match name in cluster config"), ) } - if topicConfig.Meta.Environment != clusterConfig.Meta.Environment { + if resourceMeta.Environment != clusterConfig.Meta.Environment { err = multierror.Append( err, errors.New("Topic environment does not match cluster environment"), ) } - if topicConfig.Meta.Region != clusterConfig.Meta.Region { + if resourceMeta.Region != clusterConfig.Meta.Region { err = multierror.Append( err, errors.New("Topic region does not match cluster region"), diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index 8eb3e229..d56ded09 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -62,7 +62,7 @@ func TestLoadTopicsFile(t *testing.T) { assert.Equal( t, TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "topic-test", Cluster: "test-cluster", Region: "test-region", @@ -188,6 +188,6 @@ func TestCheckConsistency(t *testing.T) { assert.NoError(t, err) assert.NoError(t, topicConfig.Validate(3)) - assert.NoError(t, CheckConsistency(topicConfig, clusterConfig)) - assert.Error(t, CheckConsistency(topicConfigNoMatch, clusterConfig)) + assert.NoError(t, CheckConsistency(topicConfig.Meta, clusterConfig)) + assert.Error(t, CheckConsistency(topicConfigNoMatch.Meta, clusterConfig)) } diff --git a/pkg/config/topic.go b/pkg/config/topic.go index 935e4af8..1318d5f0 100644 --- a/pkg/config/topic.go +++ b/pkg/config/topic.go @@ -77,23 +77,8 @@ var allPickerMethods = []PickerMethod{ // TopicConfig represents the desired configuration of a topic. type TopicConfig struct { - Meta TopicMeta `json:"meta"` - Spec TopicSpec `json:"spec"` -} - -// TopicMeta stores the (mostly immutable) metadata associated with a topic. -// Inspired by the meta structs in Kubernetes objects. -type TopicMeta struct { - Name string `json:"name"` - Cluster string `json:"cluster"` - Region string `json:"region"` - Environment string `json:"environment"` - Description string `json:"description"` - Labels map[string]string `json:"labels"` - - // Consumers is a list of consumers who are expected to consume from this - // topic. - Consumers []string `json:"consumers,omitempty"` + Meta ResourceMeta `json:"meta"` + Spec TopicSpec `json:"spec"` } // TopicSpec stores the (mutable) specification for a topic. @@ -341,7 +326,7 @@ func TopicConfigFromTopicInfo( topicInfo admin.TopicInfo, ) TopicConfig { topicConfig := TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: topicInfo.Name, Cluster: clusterConfig.Meta.Name, Region: clusterConfig.Meta.Region, diff --git a/pkg/config/topic_test.go b/pkg/config/topic_test.go index 163f4980..764010d8 100644 --- a/pkg/config/topic_test.go +++ b/pkg/config/topic_test.go @@ -19,7 +19,7 @@ func TestTopicValidate(t *testing.T) { { description: "all good any placement", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -43,7 +43,7 @@ func TestTopicValidate(t *testing.T) { { description: "all good balanced-leaders placement", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -65,7 +65,7 @@ func TestTopicValidate(t *testing.T) { { description: "all good static placement", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -90,7 +90,7 @@ func TestTopicValidate(t *testing.T) { { description: "all good static-in-rack placement", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -115,7 +115,7 @@ func TestTopicValidate(t *testing.T) { { description: "missing meta fields", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Environment: "test-environment", Description: "Bootstrapped via topicctl bootstrap", @@ -134,7 +134,7 @@ func TestTopicValidate(t *testing.T) { { description: "double-setting retention", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -158,7 +158,7 @@ func TestTopicValidate(t *testing.T) { { description: "all good double-setting local retention", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -184,7 +184,7 @@ func TestTopicValidate(t *testing.T) { { description: "setting local retention without enabling remote storage", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -209,7 +209,7 @@ func TestTopicValidate(t *testing.T) { { description: "balanced leaders invalid rack count", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -230,7 +230,7 @@ func TestTopicValidate(t *testing.T) { { description: "static placement invalid num partitions", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -254,7 +254,7 @@ func TestTopicValidate(t *testing.T) { { description: "static placement invalid replication", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -278,7 +278,7 @@ func TestTopicValidate(t *testing.T) { { description: "static-in-rack placement invalid partition count", topicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -355,7 +355,7 @@ func TestTopicConfigFromTopicInfo(t *testing.T) { Version: 1, }, expTopicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -408,7 +408,7 @@ func TestTopicConfigFromTopicInfo(t *testing.T) { Version: 1, }, expTopicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", @@ -458,7 +458,7 @@ func TestTopicConfigFromTopicInfo(t *testing.T) { Version: 1, }, expTopicConfig: TopicConfig{ - Meta: TopicMeta{ + Meta: ResourceMeta{ Name: "test-topic", Cluster: "test-cluster", Region: "test-region", From 72fbe94b496c64861ea42a68ad6067592d8e5024 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:00:43 -0500 Subject: [PATCH 075/116] consistency checking for acls --- cmd/topicctl/subcmd/create.go | 1 - pkg/config/load_test.go | 13 +- .../test-cluster/acls/acl-test-invalid.yaml | 23 ++ .../test-cluster/acls/acl-test-invalid.yaml~ | 28 ++ .../test-cluster/acls/acl-test-multi.yaml | 52 +++ .../test-cluster/acls/acl-test-multi.yaml~ | 62 ++++ .../test-cluster/acls/acl-test-no-match.yaml | 29 ++ .../testdata/test-cluster/acls/acl-test.yaml | 29 ++ .../testdata/test-cluster/acls/acl-test.yaml~ | 28 ++ .../test-cluster/acls/test-acl-no-match.yaml~ | 54 +++ pkg/create/acl.go | 133 ++++++++ pkg/create/acl.go~ | 128 +++++++ pkg/create/acl_test.go | 312 ++++++++++++++++++ pkg/create/acl_test.go~ | 144 ++++++++ 14 files changed, 1034 insertions(+), 2 deletions(-) create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-no-match.yaml create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test.yaml create mode 100644 pkg/config/testdata/test-cluster/acls/acl-test.yaml~ create mode 100644 pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ create mode 100644 pkg/create/acl.go create mode 100644 pkg/create/acl.go~ create mode 100644 pkg/create/acl_test.go create mode 100644 pkg/create/acl_test.go~ diff --git a/cmd/topicctl/subcmd/create.go b/cmd/topicctl/subcmd/create.go index 15fe66a3..353d79a5 100644 --- a/cmd/topicctl/subcmd/create.go +++ b/cmd/topicctl/subcmd/create.go @@ -130,7 +130,6 @@ func createACL( aclConfigPath string, adminClients map[string]admin.Client, ) error { - // TODO: check consistency of cluster config and ACL config clusterConfigPath, err := clusterConfigForACLCreate(aclConfigPath) if err != nil { return err diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index d56ded09..2a92623e 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -117,7 +117,7 @@ func TestLoadACLsFile(t *testing.T) { assert.Equal( t, ACLConfig{ - Meta: ACLMeta{ + Meta: ResourceMeta{ Name: "acl-test", Cluster: "test-cluster", Region: "test-region", @@ -190,4 +190,15 @@ func TestCheckConsistency(t *testing.T) { assert.NoError(t, CheckConsistency(topicConfig.Meta, clusterConfig)) assert.Error(t, CheckConsistency(topicConfigNoMatch.Meta, clusterConfig)) + + aclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test.yaml") + assert.Equal(t, 1, len(aclConfigs)) + assert.NoError(t, err) + + aclConfigsNoMatches, err := LoadACLsFile("testdata/test-cluster/acls/acl-test-no-match.yaml") + assert.Equal(t, 1, len(aclConfigsNoMatches)) + assert.NoError(t, err) + + assert.NoError(t, CheckConsistency(aclConfigs[0].Meta, clusterConfig)) + assert.Error(t, CheckConsistency(aclConfigsNoMatches[0].Meta, clusterConfig)) } diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml new file mode 100644 index 00000000..11f10ba3 --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml @@ -0,0 +1,23 @@ +meta: + name: acl-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test acl + +spec: + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefix + operations: + - read diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ new file mode 100644 index 00000000..8d2c5601 --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ @@ -0,0 +1,28 @@ +meta: + name: user-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: invalid + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefix + operations: + - read diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml b/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml new file mode 100644 index 00000000..175897b3 --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml @@ -0,0 +1,52 @@ +# This is an empty config. +--- +meta: + name: acl-test1 + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test acl + +spec: + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + operations: + - read +--- +meta: + name: acl-test2 + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test acl + +spec: + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + operations: + - read +--- +# Another empty one + diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ b/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ new file mode 100644 index 00000000..6104e23a --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ @@ -0,0 +1,62 @@ +# This is an empty config. +--- +meta: + name: user-test1 + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: simple + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + operations: + - read +--- +meta: + name: user-test2 + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: simple + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + operations: + - read +--- +# Another empty one + diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-no-match.yaml b/pkg/config/testdata/test-cluster/acls/acl-test-no-match.yaml new file mode 100644 index 00000000..02fe8617 --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test-no-match.yaml @@ -0,0 +1,29 @@ +meta: + name: acl-test-no-match + cluster: test-cluster + environment: bad-env + region: test-region + description: | + Test acl + +spec: + acls: + - resource: + type: topic + name: test-topic + patternType: literal + principal: 'User:Alice' + host: "*" + permission: allow + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + principal: 'User:Alice' + host: "*" + permission: allow + operations: + - read diff --git a/pkg/config/testdata/test-cluster/acls/acl-test.yaml b/pkg/config/testdata/test-cluster/acls/acl-test.yaml new file mode 100644 index 00000000..b590e97f --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test.yaml @@ -0,0 +1,29 @@ +meta: + name: acl-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test acl + +spec: + acls: + - resource: + type: topic + name: test-topic + patternType: literal + principal: 'User:Alice' + host: "*" + permission: allow + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + principal: 'User:Alice' + host: "*" + permission: allow + operations: + - read diff --git a/pkg/config/testdata/test-cluster/acls/acl-test.yaml~ b/pkg/config/testdata/test-cluster/acls/acl-test.yaml~ new file mode 100644 index 00000000..a75ea237 --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/acl-test.yaml~ @@ -0,0 +1,28 @@ +meta: + name: user-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test user + +spec: + authentication: + type: scram-sha-512 + password: test-password + authorization: + type: simple + acls: + - resource: + type: topic + name: test-topic + patternType: literal + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + operations: + - read diff --git a/pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ b/pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ new file mode 100644 index 00000000..97b1d0f0 --- /dev/null +++ b/pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ @@ -0,0 +1,54 @@ +meta: + name: topic-test-no-match + cluster: local-cluster + environment: bad-env + region: local-region + description: | + Test topic + +spec: + partitions: 9 + replicationFactor: 2 + retentionMinutes: 100 + placement: + strategy: static + staticAssignments: + - [3, 4] + - [5, 6] + - [2, 1] + - [2, 3] + - [5, 1] + - [1, 2] + - [1, 3] + - [5, 6] + - [2, 1] + +meta: + name: acl-test + cluster: test-cluster + environment: test-env + region: test-region + description: | + Test acl + +spec: + acls: + - resource: + type: topic + name: test-topic + patternType: literal + principal: 'User:Alice' + host: "*" + permission: allow + operations: + - read + - describe + - resource: + type: group + name: test-group + patternType: prefixed + principal: 'User:Alice' + host: "*" + permission: allow + operations: + - read diff --git a/pkg/create/acl.go b/pkg/create/acl.go new file mode 100644 index 00000000..673e7621 --- /dev/null +++ b/pkg/create/acl.go @@ -0,0 +1,133 @@ +package create + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/apply" + "github.com/segmentio/topicctl/pkg/config" + log "github.com/sirupsen/logrus" +) + +// ACLCreatorConfig contains the configuration for an ACL creator. +type ACLCreatorConfig struct { + ClusterConfig config.ClusterConfig + DryRun bool + SkipConfirm bool + ACLConfig config.ACLConfig +} + +type ACLCreator struct { + config ACLCreatorConfig + adminClient admin.Client + + clusterConfig config.ClusterConfig + aclConfig config.ACLConfig +} + +func NewACLCreator( + ctx context.Context, + adminClient admin.Client, + creatorConfig ACLCreatorConfig, +) (*ACLCreator, error) { + if !adminClient.GetSupportedFeatures().ACLs { + return nil, fmt.Errorf("ACLs are not supported by this cluster") + } + + return &ACLCreator{ + config: creatorConfig, + adminClient: adminClient, + clusterConfig: creatorConfig.ClusterConfig, + aclConfig: creatorConfig.ACLConfig, + }, nil +} + +func (a *ACLCreator) Create(ctx context.Context) error { + log.Info("Validating configs...") + if err := config.CheckConsistency(a.aclConfig.Meta, a.clusterConfig); err != nil { + return err + } + + log.Info("Checking if ACLs already exists...") + + acls := a.aclConfig.ToNewACLEntries() + + allExistingACLs := []kafka.ACLEntry{} + newACLs := []kafka.ACLEntry{} + + for _, acl := range acls { + existingACLs, err := a.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: acl.ResourceType, + ResourceNameFilter: acl.ResourceName, + ResourcePatternTypeFilter: acl.ResourcePatternType, + PrincipalFilter: acl.Principal, + HostFilter: acl.Host, + Operation: acl.Operation, + PermissionType: acl.PermissionType, + }) + if err != nil { + return fmt.Errorf("error checking for existing ACL (%v): %v", acl, err) + } + if len(existingACLs) > 0 { + allExistingACLs = append(allExistingACLs, acl) + } else { + newACLs = append(newACLs, acl) + } + } + + if len(allExistingACLs) > 0 { + log.Infof( + "Found %d existing ACLs:\n%s", + len(allExistingACLs), + formatNewACLsConfig(allExistingACLs), + ) + } + + if len(newACLs) == 0 { + log.Infof("No ACLs to create") + return nil + } + + if a.config.DryRun { + log.Infof( + "Would create ACLs with config %+v", + formatNewACLsConfig(newACLs), + ) + return nil + } + + log.Infof( + "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", + formatNewACLsConfig(newACLs), + ) + + // TODO: move confirm away from apply package + ok, _ := apply.Confirm("OK to continue?", a.config.SkipConfirm) + if !ok { + return errors.New("Stopping because of user response") + } + + log.Infof("Creating new ACLs for user with config %+v", formatNewACLsConfig(newACLs)) + + if err := a.adminClient.CreateACLs(ctx, acls); err != nil { + return fmt.Errorf("error creating new ACLs: %v", err) + } + + return nil +} + +// formatNewACLsConfig generates a pretty string representation of kafka-go +// ACL configurations. +func formatNewACLsConfig(config []kafka.ACLEntry) string { + content, err := json.MarshalIndent(config, "", " ") + if err != nil { + log.Warnf("Error marshalling ACLs config: %+v", err) + return "Error" + } + + return string(content) +} diff --git a/pkg/create/acl.go~ b/pkg/create/acl.go~ new file mode 100644 index 00000000..e345bc80 --- /dev/null +++ b/pkg/create/acl.go~ @@ -0,0 +1,128 @@ +package create + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/apply" + "github.com/segmentio/topicctl/pkg/config" + log "github.com/sirupsen/logrus" +) + +// ACLCreatorConfig contains the configuration for an ACL creator. +type ACLCreatorConfig struct { + ClusterConfig config.ClusterConfig + DryRun bool + SkipConfirm bool + ACLConfig config.ACLConfig +} + +type ACLCreator struct { + config ACLCreatorConfig + adminClient admin.Client + + clusterConfig config.ClusterConfig + aclConfig config.ACLConfig +} + +func NewACLCreator( + ctx context.Context, + adminClient admin.Client, + creatorConfig ACLCreatorConfig, +) (*ACLCreator, error) { + if !adminClient.GetSupportedFeatures().ACLs { + return nil, fmt.Errorf("ACLs are not supported by this cluster") + } + + return &ACLCreator{ + config: creatorConfig, + adminClient: adminClient, + clusterConfig: creatorConfig.ClusterConfig, + aclConfig: creatorConfig.ACLConfig, + }, nil +} + +func (a *ACLCreator) Create(ctx context.Context) error { + log.Info("Checking if ACLs already exists...") + + acls := a.aclConfig.ToNewACLEntries() + + allExistingACLs := []kafka.ACLEntry{} + newACLs := []kafka.ACLEntry{} + + for _, acl := range acls { + existingACLs, err := a.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: acl.ResourceType, + ResourceNameFilter: acl.ResourceName, + ResourcePatternTypeFilter: acl.ResourcePatternType, + PrincipalFilter: acl.Principal, + HostFilter: acl.Host, + Operation: acl.Operation, + PermissionType: acl.PermissionType, + }) + if err != nil { + return fmt.Errorf("error checking for existing ACL (%v): %v", acl, err) + } + if len(existingACLs) > 0 { + allExistingACLs = append(allExistingACLs, acl) + } else { + newACLs = append(newACLs, acl) + } + } + + if len(allExistingACLs) > 0 { + log.Infof( + "Found %d existing ACLs:\n%s", + len(allExistingACLs), + formatNewACLsConfig(allExistingACLs), + ) + } + + if len(newACLs) == 0 { + log.Infof("No ACLs to create") + return nil + } + + if a.config.DryRun { + log.Infof( + "Would create ACLs with config %+v", + formatNewACLsConfig(newACLs), + ) + return nil + } + + log.Infof( + "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", + formatNewACLsConfig(newACLs), + ) + + // TODO: move confirm away from apply package + ok, _ := apply.Confirm("OK to continue?", a.config.SkipConfirm) + if !ok { + return errors.New("Stopping because of user response") + } + + log.Infof("Creating new ACLs for user with config %+v", formatNewACLsConfig(newACLs)) + + if err := a.adminClient.CreateACLs(ctx, acls); err != nil { + return fmt.Errorf("error creating new ACLs: %v", err) + } + + return nil +} + +// formatNewACLsConfig generates a pretty string representation of kafka-go +// ACL configurations. +func formatNewACLsConfig(config []kafka.ACLEntry) string { + content, err := json.MarshalIndent(config, "", " ") + if err != nil { + log.Warnf("Error marshalling ACLs config: %+v", err) + return "Error" + } + + return string(content) +} diff --git a/pkg/create/acl_test.go b/pkg/create/acl_test.go new file mode 100644 index 00000000..59bf13e4 --- /dev/null +++ b/pkg/create/acl_test.go @@ -0,0 +1,312 @@ +package create + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/config" + "github.com/segmentio/topicctl/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestCreateNewACLs(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-create-", 6) + topicName := util.RandomString("acl-create-", 6) + + aclConfig := config.ACLConfig{ + Meta: config.ResourceMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + creator := testCreator(ctx, t, aclConfig) + defer creator.adminClient.Close() + + defer func() { + _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + err := creator.Create(ctx) + require.NoError(t, err) + acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) +} + +func TestCreateExistingACLs(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-create-", 6) + topicName := util.RandomString("acl-create-", 6) + + aclConfig := config.ACLConfig{ + Meta: config.ResourceMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + creator := testCreator(ctx, t, aclConfig) + defer creator.adminClient.Close() + + defer func() { + _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + err := creator.Create(ctx) + require.NoError(t, err) + acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) + // Run create again and make sure it is idempotent + err = creator.Create(ctx) + require.NoError(t, err) + acl, err = creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) +} + +// TODO: write this test +func TestCreateACLsDryRun(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-create-", 6) + topicName := util.RandomString("acl-create-", 6) + + aclConfig := config.ACLConfig{ + Meta: config.ResourceMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + creator := testCreator(ctx, t, aclConfig) + defer creator.adminClient.Close() + creator.config.DryRun = true + + defer func() { + _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + err := creator.Create(ctx) + require.NoError(t, err) + acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{}, acl) +} + +// TODO: write this test +func TestConsistency(t *testing.T) { + t.Fatal("not implemented") +} + +func testCreator( + ctx context.Context, + t *testing.T, + aclConfig config.ACLConfig, +) *ACLCreator { + clusterConfig := config.ClusterConfig{ + Meta: config.ClusterMeta{ + Name: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ClusterSpec{ + BootstrapAddrs: []string{util.TestKafkaAddr()}, + ZKLockPath: "/topicctl/locks", + }, + } + + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") + require.NoError(t, err) + + applier, err := NewACLCreator( + ctx, + adminClient, + ACLCreatorConfig{ + ClusterConfig: clusterConfig, + ACLConfig: aclConfig, + DryRun: false, + SkipConfirm: true, + }, + ) + require.NoError(t, err) + return applier +} diff --git a/pkg/create/acl_test.go~ b/pkg/create/acl_test.go~ new file mode 100644 index 00000000..1666076e --- /dev/null +++ b/pkg/create/acl_test.go~ @@ -0,0 +1,144 @@ +package create + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/config" + "github.com/segmentio/topicctl/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestCreateNewACLs(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-create-", 6) + topicName := util.RandomString("acl-create-", 6) + + aclConfig := config.ACLConfig{ + Meta: config.ACLMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + creator := testCreator(ctx, t, aclConfig) + defer creator.adminClient.Close() + + defer func() { + _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + err := creator.Create(ctx) + require.NoError(t, err) + acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) +} + +// TODO: write this test +func TestCreateExistingACLs(t *testing.T) { + t.Fatal("not implemented") +} + +// TODO: write this test +func TestCreateACLsDryRun(t *testing.T) { + t.Fatal("not implemented") +} + +// TODO: write this test +func TestConsistency(t *testing.T) { + t.Fatal("not implemented") +} + +func testCreator( + ctx context.Context, + t *testing.T, + aclConfig config.ACLConfig, +) *ACLCreator { + clusterConfig := config.ClusterConfig{ + Meta: config.ClusterMeta{ + Name: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ClusterSpec{ + BootstrapAddrs: []string{util.TestKafkaAddr()}, + ZKLockPath: "/topicctl/locks", + }, + } + + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") + require.NoError(t, err) + + applier, err := NewACLCreator( + ctx, + adminClient, + ACLCreatorConfig{ + ClusterConfig: clusterConfig, + ACLConfig: aclConfig, + DryRun: false, + SkipConfirm: true, + }, + ) + require.NoError(t, err) + return applier +} From c1f10df0e1fa0440ed50026b759acec9b1215702 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:03:11 -0500 Subject: [PATCH 076/116] remove emacs backups --- .gitignore | 3 + .../test-cluster/acls/acl-test-invalid.yaml~ | 28 ---- .../test-cluster/acls/acl-test-multi.yaml~ | 62 -------- .../testdata/test-cluster/acls/acl-test.yaml~ | 28 ---- .../test-cluster/acls/test-acl-no-match.yaml~ | 54 ------- pkg/create/acl.go~ | 128 ---------------- pkg/create/acl_test.go~ | 144 ------------------ 7 files changed, 3 insertions(+), 444 deletions(-) delete mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ delete mode 100644 pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ delete mode 100644 pkg/config/testdata/test-cluster/acls/acl-test.yaml~ delete mode 100644 pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ delete mode 100644 pkg/create/acl.go~ delete mode 100644 pkg/create/acl_test.go~ diff --git a/.gitignore b/.gitignore index d7298b1c..a51a3ca8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ vendor/ build/ .vscode + +# Emacs backups +*~ \ No newline at end of file diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ deleted file mode 100644 index 8d2c5601..00000000 --- a/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml~ +++ /dev/null @@ -1,28 +0,0 @@ -meta: - name: user-test - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: invalid - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefix - operations: - - read diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ b/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ deleted file mode 100644 index 6104e23a..00000000 --- a/pkg/config/testdata/test-cluster/acls/acl-test-multi.yaml~ +++ /dev/null @@ -1,62 +0,0 @@ -# This is an empty config. ---- -meta: - name: user-test1 - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: simple - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - operations: - - read ---- -meta: - name: user-test2 - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: simple - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - operations: - - read ---- -# Another empty one - diff --git a/pkg/config/testdata/test-cluster/acls/acl-test.yaml~ b/pkg/config/testdata/test-cluster/acls/acl-test.yaml~ deleted file mode 100644 index a75ea237..00000000 --- a/pkg/config/testdata/test-cluster/acls/acl-test.yaml~ +++ /dev/null @@ -1,28 +0,0 @@ -meta: - name: user-test - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: simple - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - operations: - - read diff --git a/pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ b/pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ deleted file mode 100644 index 97b1d0f0..00000000 --- a/pkg/config/testdata/test-cluster/acls/test-acl-no-match.yaml~ +++ /dev/null @@ -1,54 +0,0 @@ -meta: - name: topic-test-no-match - cluster: local-cluster - environment: bad-env - region: local-region - description: | - Test topic - -spec: - partitions: 9 - replicationFactor: 2 - retentionMinutes: 100 - placement: - strategy: static - staticAssignments: - - [3, 4] - - [5, 6] - - [2, 1] - - [2, 3] - - [5, 1] - - [1, 2] - - [1, 3] - - [5, 6] - - [2, 1] - -meta: - name: acl-test - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test acl - -spec: - acls: - - resource: - type: topic - name: test-topic - patternType: literal - principal: 'User:Alice' - host: "*" - permission: allow - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - principal: 'User:Alice' - host: "*" - permission: allow - operations: - - read diff --git a/pkg/create/acl.go~ b/pkg/create/acl.go~ deleted file mode 100644 index e345bc80..00000000 --- a/pkg/create/acl.go~ +++ /dev/null @@ -1,128 +0,0 @@ -package create - -import ( - "context" - "encoding/json" - "errors" - "fmt" - - "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/admin" - "github.com/segmentio/topicctl/pkg/apply" - "github.com/segmentio/topicctl/pkg/config" - log "github.com/sirupsen/logrus" -) - -// ACLCreatorConfig contains the configuration for an ACL creator. -type ACLCreatorConfig struct { - ClusterConfig config.ClusterConfig - DryRun bool - SkipConfirm bool - ACLConfig config.ACLConfig -} - -type ACLCreator struct { - config ACLCreatorConfig - adminClient admin.Client - - clusterConfig config.ClusterConfig - aclConfig config.ACLConfig -} - -func NewACLCreator( - ctx context.Context, - adminClient admin.Client, - creatorConfig ACLCreatorConfig, -) (*ACLCreator, error) { - if !adminClient.GetSupportedFeatures().ACLs { - return nil, fmt.Errorf("ACLs are not supported by this cluster") - } - - return &ACLCreator{ - config: creatorConfig, - adminClient: adminClient, - clusterConfig: creatorConfig.ClusterConfig, - aclConfig: creatorConfig.ACLConfig, - }, nil -} - -func (a *ACLCreator) Create(ctx context.Context) error { - log.Info("Checking if ACLs already exists...") - - acls := a.aclConfig.ToNewACLEntries() - - allExistingACLs := []kafka.ACLEntry{} - newACLs := []kafka.ACLEntry{} - - for _, acl := range acls { - existingACLs, err := a.adminClient.GetACLs(ctx, kafka.ACLFilter{ - ResourceTypeFilter: acl.ResourceType, - ResourceNameFilter: acl.ResourceName, - ResourcePatternTypeFilter: acl.ResourcePatternType, - PrincipalFilter: acl.Principal, - HostFilter: acl.Host, - Operation: acl.Operation, - PermissionType: acl.PermissionType, - }) - if err != nil { - return fmt.Errorf("error checking for existing ACL (%v): %v", acl, err) - } - if len(existingACLs) > 0 { - allExistingACLs = append(allExistingACLs, acl) - } else { - newACLs = append(newACLs, acl) - } - } - - if len(allExistingACLs) > 0 { - log.Infof( - "Found %d existing ACLs:\n%s", - len(allExistingACLs), - formatNewACLsConfig(allExistingACLs), - ) - } - - if len(newACLs) == 0 { - log.Infof("No ACLs to create") - return nil - } - - if a.config.DryRun { - log.Infof( - "Would create ACLs with config %+v", - formatNewACLsConfig(newACLs), - ) - return nil - } - - log.Infof( - "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", - formatNewACLsConfig(newACLs), - ) - - // TODO: move confirm away from apply package - ok, _ := apply.Confirm("OK to continue?", a.config.SkipConfirm) - if !ok { - return errors.New("Stopping because of user response") - } - - log.Infof("Creating new ACLs for user with config %+v", formatNewACLsConfig(newACLs)) - - if err := a.adminClient.CreateACLs(ctx, acls); err != nil { - return fmt.Errorf("error creating new ACLs: %v", err) - } - - return nil -} - -// formatNewACLsConfig generates a pretty string representation of kafka-go -// ACL configurations. -func formatNewACLsConfig(config []kafka.ACLEntry) string { - content, err := json.MarshalIndent(config, "", " ") - if err != nil { - log.Warnf("Error marshalling ACLs config: %+v", err) - return "Error" - } - - return string(content) -} diff --git a/pkg/create/acl_test.go~ b/pkg/create/acl_test.go~ deleted file mode 100644 index 1666076e..00000000 --- a/pkg/create/acl_test.go~ +++ /dev/null @@ -1,144 +0,0 @@ -package create - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/admin" - "github.com/segmentio/topicctl/pkg/config" - "github.com/segmentio/topicctl/pkg/util" - "github.com/stretchr/testify/require" -) - -func TestCreateNewACLs(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - principal := util.RandomString("User:acl-create-", 6) - topicName := util.RandomString("acl-create-", 6) - - aclConfig := config.ACLConfig{ - Meta: config.ACLMeta{ - Name: "test-acl", - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - }, - Spec: config.ACLSpec{ - ACLs: []config.ACL{ - { - Resource: config.ACLResource{ - Type: kafka.ResourceTypeTopic, - Name: topicName, - PatternType: kafka.PatternTypeLiteral, - Principal: principal, - Host: "*", - Permission: kafka.ACLPermissionTypeAllow, - }, - Operations: []kafka.ACLOperationType{ - kafka.ACLOperationTypeRead, - }, - }, - }, - }, - } - creator := testCreator(ctx, t, aclConfig) - defer creator.adminClient.Close() - - defer func() { - _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, - &kafka.DeleteACLsRequest{ - Filters: []kafka.DeleteACLsFilter{ - { - ResourceTypeFilter: kafka.ResourceTypeTopic, - ResourceNameFilter: topicName, - ResourcePatternTypeFilter: kafka.PatternTypeLiteral, - PrincipalFilter: principal, - HostFilter: "*", - PermissionType: kafka.ACLPermissionTypeAllow, - Operation: kafka.ACLOperationTypeRead, - }, - }, - }, - ) - - if err != nil { - t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) - } - }() - err := creator.Create(ctx) - require.NoError(t, err) - acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ - ResourceTypeFilter: kafka.ResourceTypeTopic, - ResourceNameFilter: topicName, - ResourcePatternTypeFilter: kafka.PatternTypeLiteral, - PrincipalFilter: principal, - HostFilter: "*", - PermissionType: kafka.ACLPermissionTypeAllow, - Operation: kafka.ACLOperationTypeRead, - }) - require.NoError(t, err) - require.Equal(t, []admin.ACLInfo{ - { - ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), - ResourceName: topicName, - PatternType: admin.PatternType(kafka.PatternTypeLiteral), - Principal: principal, - Host: "*", - Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), - PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), - }, - }, acl) -} - -// TODO: write this test -func TestCreateExistingACLs(t *testing.T) { - t.Fatal("not implemented") -} - -// TODO: write this test -func TestCreateACLsDryRun(t *testing.T) { - t.Fatal("not implemented") -} - -// TODO: write this test -func TestConsistency(t *testing.T) { - t.Fatal("not implemented") -} - -func testCreator( - ctx context.Context, - t *testing.T, - aclConfig config.ACLConfig, -) *ACLCreator { - clusterConfig := config.ClusterConfig{ - Meta: config.ClusterMeta{ - Name: "test-cluster", - Region: "test-region", - Environment: "test-environment", - }, - Spec: config.ClusterSpec{ - BootstrapAddrs: []string{util.TestKafkaAddr()}, - ZKLockPath: "/topicctl/locks", - }, - } - - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") - require.NoError(t, err) - - applier, err := NewACLCreator( - ctx, - adminClient, - ACLCreatorConfig{ - ClusterConfig: clusterConfig, - ACLConfig: aclConfig, - DryRun: false, - SkipConfirm: true, - }, - ) - require.NoError(t, err) - return applier -} From a2d4686ac0d709788152ba8ed92b26f39237fd77 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:04:30 -0500 Subject: [PATCH 077/116] remove user stuff --- cmd/topicctl/subcmd/applyuser.go | 171 ------------------ pkg/apply/applyuser.go | 160 ----------------- pkg/apply/applyuser_test.go | 135 -------------- pkg/config/user.go | 235 ------------------------ pkg/config/user_test.go | 294 ------------------------------- 5 files changed, 995 deletions(-) delete mode 100644 cmd/topicctl/subcmd/applyuser.go delete mode 100644 pkg/apply/applyuser.go delete mode 100644 pkg/apply/applyuser_test.go delete mode 100644 pkg/config/user.go delete mode 100644 pkg/config/user_test.go diff --git a/cmd/topicctl/subcmd/applyuser.go b/cmd/topicctl/subcmd/applyuser.go deleted file mode 100644 index c7457532..00000000 --- a/cmd/topicctl/subcmd/applyuser.go +++ /dev/null @@ -1,171 +0,0 @@ -package subcmd - -// import ( -// "context" -// "fmt" -// "os" -// "os/signal" -// "path/filepath" -// "syscall" - -// "github.com/segmentio/topicctl/pkg/admin" -// "github.com/segmentio/topicctl/pkg/apply" -// "github.com/segmentio/topicctl/pkg/cli" -// "github.com/segmentio/topicctl/pkg/config" -// log "github.com/sirupsen/logrus" -// "github.com/spf13/cobra" -// ) - -// var applyUserCmd = &cobra.Command{ -// Use: "apply-user [user configs]", -// Short: "apply one or more user configs", -// Args: cobra.MinimumNArgs(1), -// RunE: applyUserRun, -// } - -// func init() { -// applyUserCmd.Flags().BoolVar( -// &applyConfig.dryRun, -// "dry-run", -// false, -// "Do a dry-run", -// ) -// applyUserCmd.Flags().StringVar( -// &applyConfig.pathPrefix, -// "path-prefix", -// os.Getenv("TOPICCTL_USER_APPLY_PATH_PREFIX"), -// "Prefix for user config paths", -// ) -// applyUserCmd.Flags().BoolVar( -// &applyConfig.skipConfirm, -// "skip-confirm", -// false, -// "Skip confirmation prompts during apply process", -// ) - -// addSharedConfigOnlyFlags(applyUserCmd, &applyConfig.shared) -// RootCmd.AddCommand(applyUserCmd) -// } - -// func applyUserRun(cmd *cobra.Command, args []string) error { -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// sigChan := make(chan os.Signal, 1) -// signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) -// go func() { -// <-sigChan -// cancel() -// }() - -// // Keep a cache of the admin clients with the cluster config path as the key -// adminClients := map[string]admin.Client{} - -// defer func() { -// for _, adminClient := range adminClients { -// adminClient.Close() -// } -// }() - -// matchCount := 0 - -// for _, arg := range args { -// if applyConfig.pathPrefix != "" && !filepath.IsAbs(arg) { -// arg = filepath.Join(applyConfig.pathPrefix, arg) -// } - -// matches, err := filepath.Glob(arg) -// if err != nil { -// return err -// } - -// for _, match := range matches { -// matchCount++ -// if err := applyUser(ctx, match, adminClients); err != nil { -// return err -// } -// } -// } - -// if matchCount == 0 { -// return fmt.Errorf("No user configs match the provided args (%+v)", args) -// } - -// return nil -// } - -// func applyUser( -// ctx context.Context, -// userConfigPath string, -// adminClients map[string]admin.Client, -// ) error { -// clusterConfigPath, err := clusterConfigForUserApply(userConfigPath) -// if err != nil { -// return err -// } - -// userConfigs, err := config.LoadUsersFile(userConfigPath) -// if err != nil { -// return err -// } - -// clusterConfig, err := config.LoadClusterFile(clusterConfigPath, applyConfig.shared.expandEnv) -// if err != nil { -// return err -// } - -// adminClient, ok := adminClients[clusterConfigPath] -// if !ok { -// adminClient, err = clusterConfig.NewAdminClient( -// ctx, -// nil, -// applyConfig.dryRun, -// applyConfig.shared.saslUsername, -// applyConfig.shared.saslPassword, -// ) -// if err != nil { -// return err -// } -// adminClients[clusterConfigPath] = adminClient -// } - -// cliRunner := cli.NewCLIRunner(adminClient, log.Infof, false) - -// for _, userConfig := range userConfigs { -// // userConfig.SetDefaults() -// log.Infof( -// "Processing user %s in config %s with cluster config %s", -// userConfig.Meta.Name, -// userConfigPath, -// clusterConfigPath, -// ) - -// applierConfig := apply.UserApplierConfig{ -// DryRun: applyConfig.dryRun, -// SkipConfirm: applyConfig.skipConfirm, -// UserConfig: userConfig, -// ClusterConfig: clusterConfig, -// } - -// if err := cliRunner.ApplyUser(ctx, applierConfig); err != nil { -// return err -// } -// } - -// return nil -// } - -// // TODO: move this into a util function shared between this and apply topic -// func clusterConfigForUserApply(userConfigPath string) (string, error) { -// if applyConfig.shared.clusterConfig != "" { -// return applyConfig.shared.clusterConfig, nil -// } - -// return filepath.Abs( -// filepath.Join( -// filepath.Dir(userConfigPath), -// "..", -// "cluster.yaml", -// ), -// ) -// } diff --git a/pkg/apply/applyuser.go b/pkg/apply/applyuser.go deleted file mode 100644 index 7e3a44b5..00000000 --- a/pkg/apply/applyuser.go +++ /dev/null @@ -1,160 +0,0 @@ -package apply - -// // TODO: dry this up with the apply.go file - -// // TODO: are these structs even necessary? -// type UserApplierConfig struct { -// DryRun bool -// SkipConfirm bool -// UserConfig config.UserConfig -// ClusterConfig config.ClusterConfig -// } - -// type UserApplier struct { -// config UserApplierConfig -// adminClient admin.Client - -// clusterConfig config.ClusterConfig -// userConfig config.UserConfig -// } - -// func NewUserApplier( -// ctx context.Context, -// adminClient admin.Client, -// applierConfig UserApplierConfig, -// ) (*UserApplier, error) { -// if !adminClient.GetSupportedFeatures().Applies { -// return nil, -// errors.New( -// "Admin client does not support features needed for apply; You need to upgrade to Kafka version >2.7.0.", -// ) -// } - -// return &UserApplier{ -// config: applierConfig, -// adminClient: adminClient, -// clusterConfig: applierConfig.ClusterConfig, -// userConfig: applierConfig.UserConfig, -// }, nil -// } - -// func (u *UserApplier) Apply(ctx context.Context) error { -// log.Info("Validating configs...") - -// if err := u.clusterConfig.Validate(); err != nil { -// return fmt.Errorf("error validating cluster config: %v", err) -// } - -// if err := u.userConfig.Validate(); err != nil { -// return fmt.Errorf("error validating user config: %v", err) -// } - -// log.Info("Checking if user already exists...") - -// userInfo, err := u.adminClient.GetUsers(ctx, []string{u.userConfig.Meta.Name}) -// if err != nil { -// return fmt.Errorf("error checking if user already exists: %v", err) -// } - -// if len(userInfo) == 0 { -// err = u.applyNewUser(ctx) -// } else { -// // TODO: handle case where this returns multiple users due to multiple creds being created -// err = u.applyExistingUser(ctx, userInfo[0]) -// } - -// if err != nil { -// return fmt.Errorf("error applying existing user: %v", err) -// } - -// log.Info("Checking if ACLs already exist for this user...") - -// existingACLs, err := u.adminClient.GetACLs(ctx, kafka.ACLFilter{ -// ResourceTypeFilter: kafka.ResourceTypeAny, -// ResourcePatternTypeFilter: kafka.PatternTypeAny, -// PrincipalFilter: fmt.Sprintf("User:%s", u.userConfig.Meta.Name), -// Operation: kafka.ACLOperationTypeAny, -// PermissionType: kafka.ACLPermissionTypeAny, -// }) - -// if err != nil { -// return fmt.Errorf("error checking existing ACLs for user %s: %v", u.userConfig.Meta.Name, err) -// } - -// // TODO: pretty print these ACLs -// log.Info("Found ", len(existingACLs), " existing ACLs: ", existingACLs) - -// acls := u.userConfig.ToNewACLEntries() - -// // Compare acls and existingACLEntries - -// if len(acls) == 0 { -// return nil -// } - -// if u.config.DryRun { -// log.Infof( -// "Would create ACLs with config %+v", -// FormatNewACLsConfig(acls), -// ) -// return nil -// } - -// log.Infof( -// "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", -// FormatNewACLsConfig(acls), -// ) - -// ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) -// if !ok { -// return errors.New("Stopping because of user response") -// } - -// log.Infof("Creating new ACLs for user with config %+v", acls) - -// err = u.adminClient.CreateACLs(ctx, acls) -// if err != nil { -// return fmt.Errorf("error creating new ACLs: %v", err) -// } -// return nil -// } - -// func (u *UserApplier) applyNewUser(ctx context.Context) error { -// user, err := u.userConfig.ToNewUserScramCredentialsUpsertion() -// if err != nil { -// return fmt.Errorf("error creating UserScramCredentialsUpsertion: %v", err) -// } - -// if u.config.DryRun { -// log.Infof("Would create user with config %+v", user) -// return nil -// } - -// log.Infof( -// "It looks like this user doesn't already exists. Will create it with this config:\n%s", -// FormatNewUserConfig(user), -// ) - -// ok, _ := Confirm("OK to continue?", u.config.SkipConfirm) -// if !ok { -// return errors.New("Stopping because of user response") -// } - -// log.Infof("Creating new user with config %+v", user) - -// err = u.adminClient.UpsertUser(ctx, user) -// if err != nil { -// return fmt.Errorf("error creating new user: %v", err) -// } - -// return nil -// } - -// // TODO: support this -// func (u *UserApplier) applyExistingUser( -// ctx context.Context, -// userInfo admin.UserInfo, -// ) error { -// log.Infof("Updating existing user: %s", u.userConfig.Meta.Name) -// return nil -// } diff --git a/pkg/apply/applyuser_test.go b/pkg/apply/applyuser_test.go deleted file mode 100644 index 920775b0..00000000 --- a/pkg/apply/applyuser_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package apply - -import ( - "context" - "testing" - "time" - - "github.com/segmentio/topicctl/pkg/config" - "github.com/segmentio/topicctl/pkg/util" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TODO: write these tests -func TestApplyNewUser(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - userName := util.RandomString("apply-user-", 6) - userConfig := config.UserConfig{ - Meta: config.UserMeta{ - Name: userName, - Cluster: "test-cluster", - Region: "test-region", - Environment: "test-environment", - }, - Spec: config.UserSpec{ - Authentication: config.AuthenticationConfig{ - Type: "scram-sha-512", - Password: "test-password", - }, - Authorization: config.AuthorizationConfig{ - Type: "simple", - ACLs: []config.ACL{ - { - Resource: config.ACLResource{ - Type: "topic", - Name: "test-topic", - PatternType: "literal", - Host: "*", - }, - Operations: []string{ - "read", - "describe", - }, - }, - }, - }, - }, - } - - applier := testUserApplier(ctx, t, userConfig) - - defer applier.adminClient.Close() - err := applier.Apply(ctx) - require.NoError(t, err) - - userInfo, err := applier.adminClient.GetUsers(ctx, []string{userName}) - require.NoError(t, err) - assert.Equal(t, 1, len(userInfo)) - assert.Equal(t, userName, userInfo[0].Name) - - // TODO: login with user creds - clusterConfig := config.ClusterConfig{ - Meta: config.ClusterMeta{ - Name: "test-cluster", - Region: "test-region", - Environment: "test-environment", - }, - - Spec: config.ClusterSpec{ - BootstrapAddrs: []string{util.TestKafkaAddr()}, - SASL: config.SASLConfig{ - Enabled: true, - Mechanism: "scram-sha-512", - Username: userName, - Password: "test-password", - }, - }, - } - - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") - require.NoError(t, err) - _, err = adminClient.GetUsers(ctx, []string{userName}) - require.NoError(t, err) - // TODO: check acls - // TODO: check empty host gets coalesced to "*" -} - -func TestApplyExistingUser(t *testing.T) {} - -func TestApplyUserDryRun(t *testing.T) {} - -func testUserApplier( - ctx context.Context, - t *testing.T, - userConfig config.UserConfig, -) *UserApplier { - clusterConfig := config.ClusterConfig{ - Meta: config.ClusterMeta{ - Name: "test-cluster", - Region: "test-region", - Environment: "test-environment", - }, - - Spec: config.ClusterSpec{ - BootstrapAddrs: []string{util.TestKafkaAddr()}, - //ZKAddrs: []string{util.TestZKAddr()}, - ZKLockPath: "/topicctl/locks", - }, - } - - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") - require.NoError(t, err) - - applier, err := NewUserApplier( - ctx, - adminClient, - UserApplierConfig{ - ClusterConfig: clusterConfig, - UserConfig: userConfig, - DryRun: false, - SkipConfirm: true, - }, - ) - require.NoError(t, err) - return applier -} - -// create a function that lists all users and deletes them -func TestDeleteUser(t *testing.T) { - // create a user - - // delete the user -} diff --git a/pkg/config/user.go b/pkg/config/user.go deleted file mode 100644 index 11785474..00000000 --- a/pkg/config/user.go +++ /dev/null @@ -1,235 +0,0 @@ -package config - -// import ( -// "crypto/rand" -// "crypto/sha512" -// "errors" -// "fmt" - -// "github.com/hashicorp/go-multierror" -// "github.com/segmentio/kafka-go" -// "github.com/segmentio/topicctl/pkg/admin" -// "github.com/xdg-go/pbkdf2" -// ) - -// type UserConfig struct { -// Meta UserMeta `json:"meta"` -// Spec UserSpec `json:"spec"` -// } - -// type UserMeta struct { -// Name string `json:"name"` -// Cluster string `json:"cluster"` -// Region string `json:"region"` -// Environment string `json:"environment"` -// Description string `json:"description"` -// Labels map[string]string `json:"labels"` -// } - -// type UserSpec struct { -// Authentication AuthenticationConfig `json:"authentication,omitempty"` -// Authorization AuthorizationConfig `json:"authorization,omitempty"` -// } - -// type AuthenticationConfig struct { -// Type AuthenticationType `json:"type"` -// // TODO: extend this to a type that supports SSMRef -// Password string `json:"password"` -// } - -// type AuthenticationType string - -// const ( -// ScramSha512 AuthenticationType = "scram-sha-512" -// ) - -// var allAuthenticationTypes = []AuthenticationType{ -// ScramSha512, -// } - -// type AuthorizationConfig struct { -// Type AuthorizationType `json:"type"` -// ACLs []ACL `json:"acls,omitempty"` -// } - -// type AuthorizationType string - -// const ( -// SimpleAuthorization AuthorizationType = "simple" -// ) - -// var allAuthorizationTypes = []AuthorizationType{ -// SimpleAuthorization, -// } - -// type ACL struct { -// Resource ACLResource `json:"resource"` -// Operations []string `json:"operations"` -// } - -// // TODO: how should principal and permission type be handled? -// // principal will always be the meta name and permission type will always be allowed -// type ACLResource struct { -// Type string `json:"type"` -// Name string `json:"name"` -// PatternType string `json:"patternType"` -// Principal string `json:"principal"` -// Host string `json:"host"` -// } - -// func keys[K comparable, V any](m map[K]V) []K { -// keys := make([]K, 0, len(m)) -// for k := range m { -// keys = append(keys, k) -// } -// return keys -// } - -// var allResourceTypes = keys(admin.ResourceTypeMap) -// var allPatternTypes = keys(admin.PatternTypeMap) -// var allOperationTypes = keys(admin.AclOperationTypeMap) - -// func (u *UserConfig) SetDefaults() { -// if u.Spec.Authorization.Type == "" { -// u.Spec.Authorization.Type = SimpleAuthorization -// } -// } - -// func (u *UserConfig) Validate() error { -// // TODO: validate password types -// var err error - -// if u.Meta.Name == "" { -// err = multierror.Append(err, errors.New("Name must be set")) -// } -// if u.Meta.Cluster == "" { -// err = multierror.Append(err, errors.New("Cluster must be set")) -// } -// if u.Meta.Region == "" { -// err = multierror.Append(err, errors.New("Region must be set")) -// } -// if u.Meta.Environment == "" { -// err = multierror.Append(err, errors.New("Environment must be set")) -// } - -// authenticationTypeFound := false -// for _, authenticationType := range allAuthenticationTypes { -// if authenticationType == u.Spec.Authentication.Type { -// authenticationTypeFound = true -// } -// } - -// if !authenticationTypeFound { -// err = multierror.Append( -// err, -// fmt.Errorf("Authentication Type must be in %+v, got: %s", allAuthenticationTypes, u.Spec.Authentication.Type), -// ) -// } - -// authorizationTypeFound := false -// for _, authorizationType := range allAuthorizationTypes { -// if authorizationType == u.Spec.Authorization.Type { -// authorizationTypeFound = true -// } -// } - -// if !authorizationTypeFound { -// err = multierror.Append( -// err, -// fmt.Errorf("Authorization Type must be in %+v, got: %s", allAuthorizationTypes, u.Spec.Authorization.Type), -// ) -// } - -// for _, acl := range u.Spec.Authorization.ACLs { -// if _, ok := admin.ResourceTypeMap[acl.Resource.Type]; !ok { -// err = multierror.Append( -// err, -// fmt.Errorf("ACL Resource Type must be in %+v, got: %s", allResourceTypes, acl.Resource.Type), -// ) -// } -// if _, ok := admin.PatternTypeMap[acl.Resource.PatternType]; !ok { -// err = multierror.Append( -// err, -// fmt.Errorf("ACL Resource PatternType must be in %+v, got: %s", allPatternTypes, acl.Resource.PatternType), -// ) -// } -// for _, operation := range acl.Operations { -// if _, ok := admin.AclOperationTypeMap[operation]; !ok { -// err = multierror.Append( -// err, -// fmt.Errorf("ACL OperationType must be in %+v, got: %s", allOperationTypes, operation), -// ) -// } -// } -// } - -// return err -// } - -// const ( -// // Currently only scram-sha-512 is supported -// ScramMechanism kafka.ScramMechanism = kafka.ScramMechanismSha512 -// // Use the same default as Postgres and Strimzi for Scram iterations -// ScramIterations int = 4096 -// ) - -// func (u UserConfig) ToNewUserScramCredentialsUpsertion() (kafka.UserScramCredentialsUpsertion, error) { -// salt := make([]byte, 24) -// if _, err := rand.Read(salt); err != nil { -// return kafka.UserScramCredentialsUpsertion{}, fmt.Errorf("User %s: unable to generate salt: %v", u.Meta.Name, err) -// } -// saltedPassword := pbkdf2.Key([]byte(u.Spec.Authentication.Password), salt, ScramIterations, sha512.Size, sha512.New) - -// return kafka.UserScramCredentialsUpsertion{ -// Name: u.Meta.Name, -// Mechanism: ScramMechanism, -// Iterations: ScramIterations, -// Salt: salt, -// SaltedPassword: saltedPassword, -// }, nil -// } - -// func (u UserConfig) ToNewACLEntries() []kafka.ACLEntry { -// acls := []kafka.ACLEntry{} - -// for _, acl := range u.Spec.Authorization.ACLs { -// // Data has already been validated before calling this function so no need to check validity - -// resourceType, _ := admin.ResourceTypeMap[acl.Resource.Type] -// resourcePatternType, _ := admin.PatternTypeMap[acl.Resource.PatternType] - -// for _, operation := range acl.Operations { -// aclOperation, _ := admin.AclOperationTypeMap[operation] - -// acls = append(acls, kafka.ACLEntry{ -// ResourceType: resourceType, -// ResourceName: acl.Resource.Name, -// ResourcePatternType: resourcePatternType, -// Principal: fmt.Sprintf("User:%s", u.Meta.Name), -// Host: acl.Resource.Host, -// Operation: aclOperation, -// PermissionType: kafka.ACLPermissionTypeAllow, -// }) -// } -// } -// return acls -// } - -// func KafkaGoACLEntriesToACLs(acls []kafka.ACLEntry) []ACL { -// ACLs := []ACL{} - -// for _, acl := range acls { -// ACLs = append(ACLs, ACL{ -// Resource: ACLResource{ -// Type: acl.ResourceType.String(), -// Name: acl.ResourceName, -// PatternType: acl.ResourcePatternType.String(), -// Principal: acl.Principal, -// Host: acl.Host, -// }, -// Operations: []string{acl.Operation.String()}, -// }) -// } - -// return ACLs -// } diff --git a/pkg/config/user_test.go b/pkg/config/user_test.go deleted file mode 100644 index 475ee1c3..00000000 --- a/pkg/config/user_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package config - -// import ( -// "testing" - -// "github.com/stretchr/testify/assert" -// ) - -// func TestUserValidate(t *testing.T) { -// type testCase struct { -// description string -// userConfig UserConfig -// expError bool -// } - -// testCases := []testCase{ -// { -// description: "happy path", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// Environment: "test-environment", -// Description: "Bootstrapped via topicctl bootstrap", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "scram-sha-512", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "simple", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "topic", -// Name: "test-topic", -// PatternType: "literal", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "read", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: false, -// }, -// { -// description: "missing meta fields", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "scram-sha-512", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "simple", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "topic", -// Name: "test-topic", -// PatternType: "literal", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "read", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: true, -// }, -// { -// description: "invalid authentication type", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// Environment: "test-environment", -// Description: "Bootstrapped via topicctl bootstrap", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "invalid", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "simple", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "topic", -// Name: "test-topic", -// PatternType: "literal", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "read", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: true, -// }, -// { -// description: "invalid authorization type", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// Environment: "test-environment", -// Description: "Bootstrapped via topicctl bootstrap", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "scram-sha-512", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "invalid", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "topic", -// Name: "test-topic", -// PatternType: "literal", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "read", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: true, -// }, -// { -// description: "invalid ACL resource type", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// Environment: "test-environment", -// Description: "Bootstrapped via topicctl bootstrap", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "scram-sha-512", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "simple", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "invalid", -// Name: "test-topic", -// PatternType: "literal", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "read", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: true, -// }, -// { -// description: "invalid ACL resource pattern type", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// Environment: "test-environment", -// Description: "Bootstrapped via topicctl bootstrap", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "scram-sha-512", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "simple", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "topic", -// Name: "test-topic", -// PatternType: "invalid", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "read", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: true, -// }, -// { -// description: "invalid ACL operation type", -// userConfig: UserConfig{ -// Meta: UserMeta{ -// Name: "test-user", -// Cluster: "test-cluster", -// Region: "test-region", -// Environment: "test-environment", -// Description: "Bootstrapped via topicctl bootstrap", -// }, -// Spec: UserSpec{ -// Authentication: AuthenticationConfig{ -// Type: "scram-sha-512", -// Password: "test-password", -// }, -// Authorization: AuthorizationConfig{ -// Type: "simple", -// ACLs: []ACL{ -// { -// Resource: ACLResource{ -// Type: "topic", -// Name: "test-topic", -// PatternType: "literal", -// Principal: "User:alice", -// Host: "*", -// }, -// Operations: []string{ -// "invalid", -// "describe", -// }, -// }, -// }, -// }, -// }, -// }, -// expError: true, -// }, -// } - -// for _, testCase := range testCases { -// err := testCase.userConfig.Validate() -// if testCase.expError { -// assert.Error(t, err, testCase.description) -// } else { -// assert.NoError(t, err, testCase.description) -// } -// } -// } - -// // TODO: write this test -// func TestUserConfigFromUserInfo(t *testing.T) { -// t.Fatal("implement me") -// } - -// // TODO: write this test -// func TestACLEntryFromUser(t *testing.T) { -// t.Fatal("implement me") -// } From a7c130c5d45439404ce45f761444b6b18f5cd284 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:10:24 -0500 Subject: [PATCH 078/116] remove diff from cluster.yaml file --- examples/auth/cluster.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/auth/cluster.yaml b/examples/auth/cluster.yaml index 0972d7fb..b6c92123 100644 --- a/examples/auth/cluster.yaml +++ b/examples/auth/cluster.yaml @@ -19,6 +19,7 @@ spec: skipVerify: true sasl: enabled: true + mechanism: SCRAM-SHA-512 # As an alternative to storing these in plain text in the config (probably not super-secure), # these can also be set via: From 23b544a33b6ba8e90cd32d675ee46e65b66cc4fd Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:11:00 -0500 Subject: [PATCH 079/116] remove diff from topic file --- examples/local-cluster/topics/topic-default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/local-cluster/topics/topic-default.yaml b/examples/local-cluster/topics/topic-default.yaml index 97b5b223..a7024142 100644 --- a/examples/local-cluster/topics/topic-default.yaml +++ b/examples/local-cluster/topics/topic-default.yaml @@ -11,7 +11,7 @@ spec: replicationFactor: 2 retentionMinutes: 100 placement: - strategy: any + strategy: in-rack settings: cleanup.policy: delete max.message.bytes: 5542880 From 7c570636259d410c6abed58ab77b1c41c5113e00 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:12:55 -0500 Subject: [PATCH 080/116] remove debug log --- pkg/admin/brokerclient.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 9ba979c2..0d5b2429 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -412,7 +412,6 @@ func (c *BrokerAdminClient) GetUsers( for _, result := range resp.Results { if result.Error != nil { - log.Debugf("got here") if errors.Is(result.Error, kafka.ResourceNotFound) { log.Debugf("Skipping over user %s because it does not exist", result.User) continue From 5931b92f6d697a0b4facefefd57a736c42997ed4 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:16:00 -0500 Subject: [PATCH 081/116] smaller diff --- pkg/admin/brokerclient_test.go | 1 + pkg/admin/types.go | 16 ++--- pkg/apply/format.go | 12 ---- .../test-cluster/users/user-test-invalid.yaml | 28 --------- .../test-cluster/users/user-test-multi.yaml | 62 ------------------- .../test-cluster/users/user-test.yaml | 28 --------- 6 files changed, 9 insertions(+), 138 deletions(-) delete mode 100644 pkg/config/testdata/test-cluster/users/user-test-invalid.yaml delete mode 100644 pkg/config/testdata/test-cluster/users/user-test-multi.yaml delete mode 100644 pkg/config/testdata/test-cluster/users/user-test.yaml diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 54cf1e73..542d225c 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -770,4 +770,5 @@ func TestBrokerClientCreateACLReadOnly(t *testing.T) { err = client.CreateACLs(ctx, []kafka.ACLEntry{}) assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) + } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 1caa958f..d55a0e85 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -94,7 +94,7 @@ type ACLInfo struct { // as a Cobra flag. type ResourceType kafka.ResourceType -var ResourceTypeMap = map[string]kafka.ResourceType{ +var resourceTypeMap = map[string]kafka.ResourceType{ "any": kafka.ResourceTypeAny, "topic": kafka.ResourceTypeTopic, "group": kafka.ResourceTypeGroup, @@ -125,7 +125,7 @@ func (r *ResourceType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (r *ResourceType) Set(v string) error { - rt, ok := ResourceTypeMap[strings.ToLower(v)] + rt, ok := resourceTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) } @@ -144,7 +144,7 @@ func (r *ResourceType) Type() string { // as a Cobra flag. type PatternType kafka.PatternType -var PatternTypeMap = map[string]kafka.PatternType{ +var patternTypeMap = map[string]kafka.PatternType{ "any": kafka.PatternTypeAny, "match": kafka.PatternTypeMatch, "literal": kafka.PatternTypeLiteral, @@ -169,7 +169,7 @@ func (p *PatternType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (p *PatternType) Set(v string) error { - pt, ok := PatternTypeMap[strings.ToLower(v)] + pt, ok := patternTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "match", "literal", or "prefixed"`) } @@ -188,7 +188,7 @@ func (r *PatternType) Type() string { // as a Cobra flag. type ACLOperationType kafka.ACLOperationType -var AclOperationTypeMap = map[string]kafka.ACLOperationType{ +var aclOperationTypeMap = map[string]kafka.ACLOperationType{ "any": kafka.ACLOperationTypeAny, "all": kafka.ACLOperationTypeAll, "read": kafka.ACLOperationTypeRead, @@ -237,7 +237,7 @@ func (o *ACLOperationType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (o *ACLOperationType) Set(v string) error { - ot, ok := AclOperationTypeMap[strings.ToLower(v)] + ot, ok := aclOperationTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`) } @@ -256,7 +256,7 @@ func (o *ACLOperationType) Type() string { // as a Cobra flag. type ACLPermissionType kafka.ACLPermissionType -var AclPermissionTypeMap = map[string]kafka.ACLPermissionType{ +var aclPermissionTypeMap = map[string]kafka.ACLPermissionType{ "any": kafka.ACLPermissionTypeAny, "allow": kafka.ACLPermissionTypeAllow, "deny": kafka.ACLPermissionTypeDeny, @@ -278,7 +278,7 @@ func (p *ACLPermissionType) String() string { // Set is used by Cobra to set the value of a variable from a Cobra flag. func (p *ACLPermissionType) Set(v string) error { - pt, ok := AclPermissionTypeMap[strings.ToLower(v)] + pt, ok := aclPermissionTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "allow", or "deny"`) } diff --git a/pkg/apply/format.go b/pkg/apply/format.go index 8c3725ce..2cc6361f 100644 --- a/pkg/apply/format.go +++ b/pkg/apply/format.go @@ -25,18 +25,6 @@ func FormatNewTopicConfig(config kafka.TopicConfig) string { return string(content) } -// FormatNewUserConfig generates a pretty string representation of a kafka-go -// user config. -func FormatNewUserConfig(config kafka.UserScramCredentialsUpsertion) string { - content, err := json.MarshalIndent(config, "", " ") - if err != nil { - log.Warnf("Error marshalling user config: %+v", err) - return "Error" - } - - return string(content) -} - // FormatSettingsDiff generates a table that summarizes the differences between // the topic settings from a topic config and the settings from ZK. func FormatSettingsDiff( diff --git a/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml b/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml deleted file mode 100644 index 8d2c5601..00000000 --- a/pkg/config/testdata/test-cluster/users/user-test-invalid.yaml +++ /dev/null @@ -1,28 +0,0 @@ -meta: - name: user-test - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: invalid - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefix - operations: - - read diff --git a/pkg/config/testdata/test-cluster/users/user-test-multi.yaml b/pkg/config/testdata/test-cluster/users/user-test-multi.yaml deleted file mode 100644 index 6104e23a..00000000 --- a/pkg/config/testdata/test-cluster/users/user-test-multi.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# This is an empty config. ---- -meta: - name: user-test1 - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: simple - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - operations: - - read ---- -meta: - name: user-test2 - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: simple - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - operations: - - read ---- -# Another empty one - diff --git a/pkg/config/testdata/test-cluster/users/user-test.yaml b/pkg/config/testdata/test-cluster/users/user-test.yaml deleted file mode 100644 index a75ea237..00000000 --- a/pkg/config/testdata/test-cluster/users/user-test.yaml +++ /dev/null @@ -1,28 +0,0 @@ -meta: - name: user-test - cluster: test-cluster - environment: test-env - region: test-region - description: | - Test user - -spec: - authentication: - type: scram-sha-512 - password: test-password - authorization: - type: simple - acls: - - resource: - type: topic - name: test-topic - patternType: literal - operations: - - read - - describe - - resource: - type: group - name: test-group - patternType: prefixed - operations: - - read From 3d52f5a11280d18f4b441391e186558f020a52d7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:17:02 -0500 Subject: [PATCH 082/116] remove completed todos --- pkg/create/acl_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/create/acl_test.go b/pkg/create/acl_test.go index 59bf13e4..182257a3 100644 --- a/pkg/create/acl_test.go +++ b/pkg/create/acl_test.go @@ -199,7 +199,6 @@ func TestCreateExistingACLs(t *testing.T) { }, acl) } -// TODO: write this test func TestCreateACLsDryRun(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -272,11 +271,6 @@ func TestCreateACLsDryRun(t *testing.T) { require.Equal(t, []admin.ACLInfo{}, acl) } -// TODO: write this test -func TestConsistency(t *testing.T) { - t.Fatal("not implemented") -} - func testCreator( ctx context.Context, t *testing.T, From d0f0ec99357cb1c52031acf14f2c43a37f005e9d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:18:14 -0500 Subject: [PATCH 083/116] remove unused error helper --- pkg/util/error.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/util/error.go b/pkg/util/error.go index d721fffb..41cdfb6c 100644 --- a/pkg/util/error.go +++ b/pkg/util/error.go @@ -49,18 +49,3 @@ func AlterPartitionReassignmentsRequestAssignmentError(results []kafka.AlterPart } return nil } - -func DescribeUserScramCredentialsResponseResultsError(results []kafka.DescribeUserScramCredentialsResponseResult) error { - errors := map[string]error{} - var hasErrors bool - for _, result := range results { - if result.Error != nil { - hasErrors = true - errors[result.User] = result.Error - } - } - if hasErrors { - return fmt.Errorf("%+v", errors) - } - return nil -} From 8400a73b1a45f3d1286875c5952c19906a7a27b4 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:20:11 -0500 Subject: [PATCH 084/116] add missing meta file --- pkg/config/meta.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 pkg/config/meta.go diff --git a/pkg/config/meta.go b/pkg/config/meta.go new file mode 100644 index 00000000..c56d66d2 --- /dev/null +++ b/pkg/config/meta.go @@ -0,0 +1,16 @@ +package config + +// ResourceMeta stores the (mostly immutable) metadata associated with a resource. +// Inspired by the meta structs in Kubernetes objects. +type ResourceMeta struct { + Name string `json:"name"` + Cluster string `json:"cluster"` + Region string `json:"region"` + Environment string `json:"environment"` + Description string `json:"description"` + Labels map[string]string `json:"labels"` + + // Consumers is a list of consumers who are expected to consume from this + // topic. + Consumers []string `json:"consumers,omitempty"` +} From 1e7cc3b1245d1a2285ebf29529d19e523ff12fcd Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:28:40 -0500 Subject: [PATCH 085/116] skip ACL tests when ACLs cannot be used due to kafka version limitations --- pkg/create/acl_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/create/acl_test.go b/pkg/create/acl_test.go index 182257a3..0797fb81 100644 --- a/pkg/create/acl_test.go +++ b/pkg/create/acl_test.go @@ -14,6 +14,10 @@ import ( ) func TestCreateNewACLs(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -95,6 +99,10 @@ func TestCreateNewACLs(t *testing.T) { } func TestCreateExistingACLs(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -200,6 +208,10 @@ func TestCreateExistingACLs(t *testing.T) { } func TestCreateACLsDryRun(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() From 24fffd0bc011fa38792f8b1e10a70419ac70f4a2 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:31:52 -0500 Subject: [PATCH 086/116] fix loadacls test --- pkg/config/load_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index 2a92623e..1a0ce76c 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -1,7 +1,6 @@ package config import ( - "fmt" "os" "testing" @@ -110,7 +109,6 @@ func TestLoadTopicsFile(t *testing.T) { func TestLoadACLsFile(t *testing.T) { aclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test.yaml") require.NoError(t, err) - fmt.Println(aclConfigs) assert.Equal(t, 1, len(aclConfigs)) aclConfig := aclConfigs[0] @@ -131,6 +129,9 @@ func TestLoadACLsFile(t *testing.T) { Type: kafka.ResourceTypeTopic, Name: "test-topic", PatternType: kafka.PatternTypeLiteral, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, }, Operations: []kafka.ACLOperationType{ kafka.ACLOperationTypeRead, @@ -142,6 +143,9 @@ func TestLoadACLsFile(t *testing.T) { Type: kafka.ResourceTypeGroup, Name: "test-group", PatternType: kafka.PatternTypePrefixed, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, }, Operations: []kafka.ACLOperationType{ kafka.ACLOperationTypeRead, From 32a94903267c9fcf9fcdcf29051db9e12980b6af Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 15 Nov 2023 16:33:36 -0500 Subject: [PATCH 087/116] add more todos --- pkg/config/acl.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/config/acl.go b/pkg/config/acl.go index 189912fe..779eabdb 100644 --- a/pkg/config/acl.go +++ b/pkg/config/acl.go @@ -45,3 +45,7 @@ func (a ACLConfig) ToNewACLEntries() []kafka.ACLEntry { } return acls } + +// TODO: Set defaults for missing values + +// TODO: Validate fields cannot be empty From 39349ad5e0ab5b72fc8b5104d6096a6a25d9e492 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 11:43:11 -0500 Subject: [PATCH 088/116] add validation and set defaults --- pkg/config/acl.go | 46 ++++++- pkg/config/acl_test.go | 267 ++++++++++++++++++++++++++++++++++++++++ pkg/config/meta.go | 24 ++++ pkg/config/meta_test.go | 47 +++++++ pkg/config/topic.go | 13 +- 5 files changed, 383 insertions(+), 14 deletions(-) create mode 100644 pkg/config/acl_test.go create mode 100644 pkg/config/meta_test.go diff --git a/pkg/config/acl.go b/pkg/config/acl.go index 779eabdb..16e74848 100644 --- a/pkg/config/acl.go +++ b/pkg/config/acl.go @@ -1,6 +1,9 @@ package config import ( + "errors" + + "github.com/hashicorp/go-multierror" "github.com/segmentio/kafka-go" ) @@ -46,6 +49,45 @@ func (a ACLConfig) ToNewACLEntries() []kafka.ACLEntry { return acls } -// TODO: Set defaults for missing values +// SetDeaults sets the default host and permission for each ACL in an ACL config +// if these aren't set +func (a *ACLConfig) SetDefaults() { + for i, acl := range a.Spec.ACLs { + if acl.Resource.Host == "" { + a.Spec.ACLs[i].Resource.Host = "*" + } + if acl.Resource.Permission == kafka.ACLPermissionTypeUnknown { + a.Spec.ACLs[i].Resource.Permission = kafka.ACLPermissionTypeAllow + } + } +} + +// Validate evaluates whether the ACL config is valid. +func (a *ACLConfig) Validate() error { + var err error + + err = a.Meta.Validate() + + for _, acl := range a.Spec.ACLs { + if acl.Resource.Type == kafka.ResourceTypeUnknown { + err = multierror.Append(err, errors.New("ACL resource type cannot be unknown")) + } + if acl.Resource.Name == "" { + err = multierror.Append(err, errors.New("ACL resource name cannot be empty")) + } + if acl.Resource.PatternType == kafka.PatternTypeUnknown { + err = multierror.Append(err, errors.New("ACL resource pattern type cannot be unknown")) + } + if acl.Resource.Principal == "" { + err = multierror.Append(err, errors.New("ACL resource principal cannot be empty")) + } + + for _, operation := range acl.Operations { + if operation == kafka.ACLOperationTypeUnknown { + err = multierror.Append(err, errors.New("ACL operation cannot be unknown")) + } + } + } -// TODO: Validate fields cannot be empty + return err +} diff --git a/pkg/config/acl_test.go b/pkg/config/acl_test.go new file mode 100644 index 00000000..9f3c6a90 --- /dev/null +++ b/pkg/config/acl_test.go @@ -0,0 +1,267 @@ +package config + +import ( + "testing" + + "github.com/segmentio/kafka-go" + "github.com/stretchr/testify/assert" +) + +func TestACLValidate(t *testing.T) { + type testCase struct { + description string + aclConfig ACLConfig + expError bool + } + + testCases := []testCase{ + { + description: "valid ACL config", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: "test-topic", + PatternType: kafka.PatternTypeLiteral, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + expError: false, + }, + { + description: "unknown resource type", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeUnknown, + Name: "test-topic", + PatternType: kafka.PatternTypeLiteral, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "empty resource name", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + PatternType: kafka.PatternTypeLiteral, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "unknown resource pattern type", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + PatternType: kafka.PatternTypeUnknown, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "empty principal", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + PatternType: kafka.PatternTypeUnknown, + Principal: "", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + expError: true, + }, + { + description: "unknown ACL operation type", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + PatternType: kafka.PatternTypeUnknown, + Principal: "", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeUnknown, + }, + }, + }, + }, + }, + expError: true, + }, + } + + for _, testCase := range testCases { + testCase.aclConfig.SetDefaults() + err := testCase.aclConfig.Validate() + if testCase.expError { + assert.Error(t, err, testCase.description) + } else { + assert.NoError(t, err, testCase.description) + } + } +} + +func TestACLSetDefaults(t *testing.T) { + type testCase struct { + description string + aclConfig ACLConfig + expConfig ACLConfig + } + + testCases := []testCase{ + { + description: "set defaults", + aclConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: "test-topic", + PatternType: kafka.PatternTypeLiteral, + Principal: "User:Alice", + Permission: kafka.ACLPermissionTypeUnknown, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + expConfig: ACLConfig{ + Meta: ResourceMeta{ + Name: "acl-test", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-env", + }, + Spec: ACLSpec{ + ACLs: []ACL{ + { + Resource: ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: "test-topic", + PatternType: kafka.PatternTypeLiteral, + Principal: "User:Alice", + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + testCase.aclConfig.SetDefaults() + assert.Equal(t, testCase.expConfig, testCase.aclConfig, testCase.description) + } +} diff --git a/pkg/config/meta.go b/pkg/config/meta.go index c56d66d2..8415ab59 100644 --- a/pkg/config/meta.go +++ b/pkg/config/meta.go @@ -1,5 +1,11 @@ package config +import ( + "errors" + + "github.com/hashicorp/go-multierror" +) + // ResourceMeta stores the (mostly immutable) metadata associated with a resource. // Inspired by the meta structs in Kubernetes objects. type ResourceMeta struct { @@ -14,3 +20,21 @@ type ResourceMeta struct { // topic. Consumers []string `json:"consumers,omitempty"` } + +// Validate evalutes whether the ResourceMeta is valid. +func (rm *ResourceMeta) Validate() error { + var err error + if rm.Name == "" { + err = multierror.Append(err, errors.New("Name must be set")) + } + if rm.Cluster == "" { + err = multierror.Append(err, errors.New("Cluster must be set")) + } + if rm.Region == "" { + err = multierror.Append(err, errors.New("Region must be set")) + } + if rm.Environment == "" { + err = multierror.Append(err, errors.New("Environment must be set")) + } + return err +} diff --git a/pkg/config/meta_test.go b/pkg/config/meta_test.go new file mode 100644 index 00000000..0d20cd51 --- /dev/null +++ b/pkg/config/meta_test.go @@ -0,0 +1,47 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMetaValidate(t *testing.T) { + type testCase struct { + description string + meta ResourceMeta + expError bool + } + + testCases := []testCase{ + { + description: "valid meta", + meta: ResourceMeta{ + Name: "test-topic", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + Description: "test-description", + }, + expError: false, + }, + { + description: "meta missing fields", + meta: ResourceMeta{ + Name: "test-topic", + Environment: "test-environment", + Description: "Bootstrapped via topicctl bootstrap", + }, + expError: true, + }, + } + + for _, testCase := range testCases { + err := testCase.meta.Validate() + if testCase.expError { + assert.Error(t, err, testCase.description) + } else { + assert.NoError(t, err, testCase.description) + } + } +} diff --git a/pkg/config/topic.go b/pkg/config/topic.go index 1318d5f0..370b877e 100644 --- a/pkg/config/topic.go +++ b/pkg/config/topic.go @@ -166,18 +166,7 @@ func (t *TopicConfig) SetDefaults() { func (t TopicConfig) Validate(numRacks int) error { var err error - if t.Meta.Name == "" { - err = multierror.Append(err, errors.New("Name must be set")) - } - if t.Meta.Cluster == "" { - err = multierror.Append(err, errors.New("Cluster must be set")) - } - if t.Meta.Region == "" { - err = multierror.Append(err, errors.New("Region must be set")) - } - if t.Meta.Environment == "" { - err = multierror.Append(err, errors.New("Environment must be set")) - } + err = t.Meta.Validate() if t.Spec.Partitions <= 0 { err = multierror.Append(err, errors.New("Partitions must be a positive number")) From 0cd4dcfba7dbcd37e7cf9ce8b9ea647b1d732759 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 11:47:51 -0500 Subject: [PATCH 089/116] don't use ioutil --- pkg/config/load.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/config/load.go b/pkg/config/load.go index 6245313f..f79200b6 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "regexp" @@ -90,7 +89,7 @@ func LoadTopicBytes(contents []byte) (TopicConfig, error) { // LoadACLsFile loads one or more ACLConfigs from a path to a YAML file. func LoadACLsFile(path string) ([]ACLConfig, error) { - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return nil, err } From 41981eba3ebe4783602d111f37940e2c780fb76b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 11:51:52 -0500 Subject: [PATCH 090/116] move confirm to util package --- cmd/topicctl/subcmd/tester.go | 6 +++--- pkg/apply/apply.go | 18 +++++++++--------- pkg/config/load_test.go | 2 -- pkg/create/acl.go | 5 ++--- pkg/{apply => util}/confirm.go | 2 +- 5 files changed, 15 insertions(+), 18 deletions(-) rename pkg/{apply => util}/confirm.go (98%) diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index 4c75a9de..03bd18eb 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -10,7 +10,7 @@ import ( "time" "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/apply" + "github.com/segmentio/topicctl/pkg/util" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -104,7 +104,7 @@ func runTestReader(ctx context.Context) error { testerConfig.readConsumer, ) - ok, _ := apply.Confirm("OK to continue?", false) + ok, _ := util.Confirm("OK to continue?", false) if !ok { return errors.New("Stopping because of user response") } @@ -153,7 +153,7 @@ func runTestWriter(ctx context.Context) error { testerConfig.writeRate, ) - ok, _ := apply.Confirm("OK to continue?", false) + ok, _ := util.Confirm("OK to continue?", false) if !ok { return errors.New("Stopping because of user response") } diff --git a/pkg/apply/apply.go b/pkg/apply/apply.go index 76e54f0f..25b0084b 100644 --- a/pkg/apply/apply.go +++ b/pkg/apply/apply.go @@ -163,7 +163,7 @@ func (t *TopicApplier) applyNewTopic(ctx context.Context) error { FormatNewTopicConfig(newTopicConfig), ) - ok, _ := Confirm("OK to continue?", t.config.SkipConfirm) + ok, _ := util.Confirm("OK to continue?", t.config.SkipConfirm) if !ok { return errors.New("Stopping because of user response") } @@ -283,7 +283,7 @@ func (t *TopicApplier) checkExistingState( if t.config.DryRun { log.Infof("Skipping update because dryRun is set to true") } else { - ok, err := Confirm("OK to remove these?", t.config.SkipConfirm) + ok, err := util.Confirm("OK to remove these?", t.config.SkipConfirm) if err != nil { return err } else if !ok { @@ -326,7 +326,7 @@ func (t *TopicApplier) checkExistingState( if t.config.DryRun { log.Infof("Skipping update because dryRun is set to true") } else { - ok, err := Confirm("OK to remove broker throttles?", t.config.SkipConfirm) + ok, err := util.Confirm("OK to remove broker throttles?", t.config.SkipConfirm) if err != nil { return err } else if !ok { @@ -413,7 +413,7 @@ func (t *TopicApplier) updateSettings( return nil } - ok, _ := Confirm( + ok, _ := util.Confirm( "OK to update to the new values in the topic config?", t.config.SkipConfirm, ) @@ -576,7 +576,7 @@ func (t *TopicApplier) updatePartitionsHelper( return nil } - ok, _ := Confirm("OK to apply?", t.config.SkipConfirm) + ok, _ := util.Confirm("OK to apply?", t.config.SkipConfirm) if !ok { return errors.New("Stopping because of user response") } @@ -678,7 +678,7 @@ func (t *TopicApplier) updatePlacement( desiredPlacement, ) - ok, _ := Confirm( + ok, _ := util.Confirm( fmt.Sprintf("OK to apply %s despite having unbalanced leaders?", desiredPlacement), t.config.SkipConfirm || t.config.DryRun, ) @@ -841,7 +841,7 @@ func (t *TopicApplier) updatePlacementRunner( log.Warnf("Autocontinue flag detected, user will not be prompted each round") } - ok, _ := Confirm("OK to apply?", t.config.SkipConfirm) + ok, _ := util.Confirm("OK to apply?", t.config.SkipConfirm) if !ok { return errors.New("Stopping because of user response") } @@ -917,7 +917,7 @@ func (t *TopicApplier) updatePlacementRunner( if t.config.AutoContinueRebalance { log.Infof("Autocontinuing to next round") } else { - ok, _ := Confirm("OK to continue?", t.config.SkipConfirm) + ok, _ := util.Confirm("OK to continue?", t.config.SkipConfirm) if !ok { return errors.New("Stopping because of user response") } @@ -1249,7 +1249,7 @@ func (t *TopicApplier) updateLeaders( batchSize = len(wrongLeaders) } - ok, _ := Confirm( + ok, _ := util.Confirm( fmt.Sprintf( "OK to run leader elections (in batches of %d partitions each) ?", batchSize, diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index 1a0ce76c..66bc36d3 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -105,7 +105,6 @@ func TestLoadTopicsFile(t *testing.T) { assert.Equal(t, "topic-test2", topicConfigs[1].Meta.Name) } -// TODO: write this test func TestLoadACLsFile(t *testing.T) { aclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test.yaml") require.NoError(t, err) @@ -159,7 +158,6 @@ func TestLoadACLsFile(t *testing.T) { invalidAclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test-invalid.yaml") assert.Equal(t, 0, len(invalidAclConfigs)) - // TODO: improve this error checking and make sure the error is informative enough require.Error(t, err) multiAclConfigs, err := LoadACLsFile("testdata/test-cluster/acls/acl-test-multi.yaml") diff --git a/pkg/create/acl.go b/pkg/create/acl.go index 673e7621..425d8b5d 100644 --- a/pkg/create/acl.go +++ b/pkg/create/acl.go @@ -8,8 +8,8 @@ import ( "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/admin" - "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/config" + "github.com/segmentio/topicctl/pkg/util" log "github.com/sirupsen/logrus" ) @@ -105,8 +105,7 @@ func (a *ACLCreator) Create(ctx context.Context) error { formatNewACLsConfig(newACLs), ) - // TODO: move confirm away from apply package - ok, _ := apply.Confirm("OK to continue?", a.config.SkipConfirm) + ok, _ := util.Confirm("OK to continue?", a.config.SkipConfirm) if !ok { return errors.New("Stopping because of user response") } diff --git a/pkg/apply/confirm.go b/pkg/util/confirm.go similarity index 98% rename from pkg/apply/confirm.go rename to pkg/util/confirm.go index 62e54787..101d5435 100644 --- a/pkg/apply/confirm.go +++ b/pkg/util/confirm.go @@ -1,4 +1,4 @@ -package apply +package util import ( "fmt" From a6d7a5eff6f47f11550967c4e3a94ba02871d20c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 11:53:42 -0500 Subject: [PATCH 091/116] move confirm to util package --- cmd/topicctl/subcmd/reset.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index 831e049d..acd9a049 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -6,9 +6,9 @@ import ( "fmt" "strconv" - "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/cli" "github.com/segmentio/topicctl/pkg/groups" + "github.com/segmentio/topicctl/pkg/util" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -167,7 +167,7 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { "Please ensure that all other consumers are stopped, otherwise the reset might be overridden.", ) - ok, _ := apply.Confirm("OK to continue?", false) + ok, _ := util.Confirm("OK to continue?", false) if !ok { return errors.New("Stopping because of user response") } From 3b4fd43ec8994b3e0de9445d3889e65a90b54c1a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 12:02:27 -0500 Subject: [PATCH 092/116] add create to README --- README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/README.md b/README.md index 7e77dcc6..b0623fa9 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,15 @@ The `check` command validates that each topic config has the correct fields set consistent with the associated cluster config. Unless `--validate-only` is set, it then checks the topic config against the state of the topic in the corresponding cluster. +#### create +``` +topicctl create [flags] [command] +``` + +The `create` command creates resources in the cluster from a configuration file. +Currently, only ACLs are supported. The create command is separate from the apply +command as it is intended for usage with immutable resources managed by topicctl. + #### get ``` @@ -419,6 +428,47 @@ This subcommand will not rebalance a topic if: 1. a topic's `retention.ms` in the kafka cluster does not match the topic's `retentionMinutes` setting in the topic config 1. a topic does not exist in the kafka cluster +### ACLs + +Sets of ACLs can be configured in a YAML file. The following is an +annotated example: + +```yaml +meta: + name: acls-test # Name of the group of ACLs + cluster: my-cluster # Name of the cluster + environment: stage # Environment of the cluster + region: us-west-2 # Region of the cluster + description: | # Free-text description of the topic (optional) + Test topic in my-cluster. + labels: # Custom key-value pairs purposed for ACL bookkeeping (optional) + key1: value1 + key2: value2 + +spec: + acls: + - resource: + type: topic # Type of resource (topic, group, cluster, etc.) + name: test-topic # Name of the resource to apply an ACL to + patternType: literal # Type of pattern (literal, prefixed, etc.) + principal: User:my-user # Principal to apply the ACL to + host: * # Host to apply the ACL to + permission: allow # Permission to apply (allow, deny) + operations: # List of operations to use for the ACLs + - read + - describe +``` + +The `cluster`, `environment`, and `region` fields are used for matching +against a cluster config and double-checking that the cluster we're applying +in is correct; they don't appear in any API calls. + +See the [Kafka documentation](https://kafka.apache.org/documentation/#security_authz_primitives) +for more details on the parameters that can be set in the `acls` field. + +Multiple groups of ACLs can be included in the same file, separated by `---` lines, provided +that they reference the same cluster. + ## Tool safety The `bootstrap`, `get`, `repl`, and `tail` subcommands are read-only and should never make @@ -441,6 +491,9 @@ The `apply` subcommand can make changes, but under the following conditions: The `reset-offsets` command can also make changes in the cluster and should be used carefully. +The `create` command can be used to create new resources in the cluster. It cannot be used with +mutuable resources. + ### Idempotency Apply runs are designed to be idemponent- the effects should be the same no matter how many From b1b5f211b67e86b93b32c8e3baeb0b0f87b51366 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 14:29:13 -0500 Subject: [PATCH 093/116] use validation and setdefaults --- cmd/topicctl/subcmd/create.go | 1 + pkg/create/acl.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/cmd/topicctl/subcmd/create.go b/cmd/topicctl/subcmd/create.go index 353d79a5..062217b1 100644 --- a/cmd/topicctl/subcmd/create.go +++ b/cmd/topicctl/subcmd/create.go @@ -163,6 +163,7 @@ func createACL( cliRunner := cli.NewCLIRunner(adminClient, log.Infof, false) for _, aclConfig := range aclConfigs { + aclConfig.SetDefaults() log.Infof( "Processing ACL %s in config %s with cluster config %s", aclConfig.Meta.Name, diff --git a/pkg/create/acl.go b/pkg/create/acl.go index 425d8b5d..882524c7 100644 --- a/pkg/create/acl.go +++ b/pkg/create/acl.go @@ -48,6 +48,15 @@ func NewACLCreator( func (a *ACLCreator) Create(ctx context.Context) error { log.Info("Validating configs...") + + if err := a.clusterConfig.Validate(); err != nil { + return err + } + + if err := a.aclConfig.Validate(); err != nil { + return err + } + if err := config.CheckConsistency(a.aclConfig.Meta, a.clusterConfig); err != nil { return err } From d749523aac42981a01a0803836eb8e5bece4b903 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 14:29:25 -0500 Subject: [PATCH 094/116] add example acl --- examples/auth/acls/acl-default.yaml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 examples/auth/acls/acl-default.yaml diff --git a/examples/auth/acls/acl-default.yaml b/examples/auth/acls/acl-default.yaml new file mode 100644 index 00000000..d28898b4 --- /dev/null +++ b/examples/auth/acls/acl-default.yaml @@ -0,0 +1,31 @@ +meta: + name: acl-default + cluster: local-cluster-auth + environment: local-env + region: local-region + description: | + This is a default ACL for the local cluster. + It grants read and describe access to the topic `my-topic` and read access to the group `my-group` + to the user `default`. + +spec: + acls: + - resource: + type: topic + name: my-topic + patternType: literal + principal: 'User:default' + host: '*' + permission: allow + operations: + - Read + - Describe + - resource: + type: group + name: my-group + patternType: prefixed + principal: 'User:default' + host: '*' + permission: allow + operations: + - Read From 67f717816f2e1ed6815fd82da4d95947cf364115 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 14:32:30 -0500 Subject: [PATCH 095/116] fix formatting in readme --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b0623fa9..7a408a3f 100644 --- a/README.md +++ b/README.md @@ -448,15 +448,15 @@ meta: spec: acls: - resource: - type: topic # Type of resource (topic, group, cluster, etc.) - name: test-topic # Name of the resource to apply an ACL to - patternType: literal # Type of pattern (literal, prefixed, etc.) - principal: User:my-user # Principal to apply the ACL to - host: * # Host to apply the ACL to - permission: allow # Permission to apply (allow, deny) + type: topic # Type of resource (topic, group, cluster, etc.) + name: test-topic # Name of the resource to apply an ACL to + patternType: literal # Type of pattern (literal, prefixed, etc.) + principal: User:my-user # Principal to apply the ACL to + host: * # Host to apply the ACL to + permission: allow # Permission to apply (allow, deny) operations: # List of operations to use for the ACLs - - read - - describe + - read + - describe ``` The `cluster`, `environment`, and `region` fields are used for matching From 3ae74de1a78b6b31d720c1bed6f6a5395eadbb8b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 14:41:17 -0500 Subject: [PATCH 096/116] use released version of kafka-go --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index cae7ae6e..66f80b7d 100644 --- a/go.mod +++ b/go.mod @@ -11,13 +11,12 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.45-0.20231030174323-c6378c391a97 + github.com/segmentio/kafka-go v0.4.45 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 - github.com/xdg-go/pbkdf2 v1.0.0 golang.org/x/crypto v0.14.0 ) @@ -38,6 +37,7 @@ require ( github.com/pkg/term v0.0.0-20200520122047-c3ffed290a03 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/go.sum b/go.sum index da752a0f..20b055b5 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= -github.com/segmentio/kafka-go v0.4.45-0.20231030174323-c6378c391a97 h1:vKYoioQZ7SgGcES2pKoNq7zV8ncKNvblHp+0O+dOeI0= -github.com/segmentio/kafka-go v0.4.45-0.20231030174323-c6378c391a97/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E= +github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= From c7db718c3cd6c50279c76afb777cfda9108d99bc Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 16:15:17 -0500 Subject: [PATCH 097/116] fix spelling --- pkg/config/acl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/acl.go b/pkg/config/acl.go index 16e74848..2bb336ff 100644 --- a/pkg/config/acl.go +++ b/pkg/config/acl.go @@ -49,7 +49,7 @@ func (a ACLConfig) ToNewACLEntries() []kafka.ACLEntry { return acls } -// SetDeaults sets the default host and permission for each ACL in an ACL config +// SetDefaults sets the default host and permission for each ACL in an ACL config // if these aren't set func (a *ACLConfig) SetDefaults() { for i, acl := range a.Spec.ACLs { From 1f8d7b42e108c3eec0f5c373a998614b0367155e Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 16:17:03 -0500 Subject: [PATCH 098/116] make invalid field more obvious --- pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml index 11f10ba3..52ffd971 100644 --- a/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml +++ b/pkg/config/testdata/test-cluster/acls/acl-test-invalid.yaml @@ -18,6 +18,6 @@ spec: - resource: type: group name: test-group - patternType: prefix + patternType: invalid operations: - read From 23f79791fb30233da7bf9404e9d079d1e35263ea Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 16 Nov 2023 16:30:52 -0500 Subject: [PATCH 099/116] fix dryrun and skip confirm --- cmd/topicctl/subcmd/create.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/topicctl/subcmd/create.go b/cmd/topicctl/subcmd/create.go index 062217b1..09a1299b 100644 --- a/cmd/topicctl/subcmd/create.go +++ b/cmd/topicctl/subcmd/create.go @@ -33,19 +33,19 @@ type createCmdConfig struct { var createConfig createCmdConfig func init() { - createCmd.Flags().BoolVar( + createCmd.PersistentFlags().BoolVar( &createConfig.dryRun, "dry-run", false, "Do a dry-run", ) - createCmd.Flags().StringVar( + createCmd.PersistentFlags().StringVar( &createConfig.pathPrefix, "path-prefix", os.Getenv("TOPICCTL_ACL_PATH_PREFIX"), "Prefix for ACL config paths", ) - createCmd.Flags().BoolVar( + createCmd.PersistentFlags().BoolVar( &createConfig.skipConfirm, "skip-confirm", false, @@ -150,9 +150,9 @@ func createACL( adminClient, err = clusterConfig.NewAdminClient( ctx, nil, - applyConfig.dryRun, - applyConfig.shared.saslUsername, - applyConfig.shared.saslPassword, + createConfig.dryRun, + createConfig.shared.saslUsername, + createConfig.shared.saslPassword, ) if err != nil { return err @@ -172,8 +172,8 @@ func createACL( ) creatorConfig := create.ACLCreatorConfig{ - DryRun: applyConfig.dryRun, - SkipConfirm: applyConfig.skipConfirm, + DryRun: createConfig.dryRun, + SkipConfirm: createConfig.skipConfirm, ACLConfig: aclConfig, ClusterConfig: clusterConfig, } From 3924172a82a9f5354fd52884e3fafeab915d50b1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 21 Nov 2023 10:50:24 -0500 Subject: [PATCH 100/116] stub out delete cli and implement admin --- cmd/topicctl/subcmd/delete.go | 65 +++++++++++++++++++++++ pkg/admin/brokerclient.go | 22 ++++++++ pkg/admin/brokerclient_test.go | 96 ++++++++++++++++++++++++++++++++++ pkg/admin/client.go | 6 +++ pkg/admin/zkclient.go | 7 +++ pkg/admin/zkclient_test.go | 15 ++++++ 6 files changed, 211 insertions(+) create mode 100644 cmd/topicctl/subcmd/delete.go diff --git a/cmd/topicctl/subcmd/delete.go b/cmd/topicctl/subcmd/delete.go new file mode 100644 index 00000000..eb92e605 --- /dev/null +++ b/cmd/topicctl/subcmd/delete.go @@ -0,0 +1,65 @@ +package subcmd + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/cli" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var deleteCmd = &cobra.Command{ + Use: "delete [resource type]", + Short: "delete instances of a particular type", + Long: strings.Join( + []string{ + "Deletes instances of a particular type.", + }, + "\n", + ), + PersistentPreRunE: deletePreRun, +} + +type deleteCmdConfig struct { + shared sharedOptions +} + +var deleteConfig deleteCmdConfig + +func init() { + addSharedFlags(deleteCmd, &deleteConfig.shared) + deleteCmd.AddCommand( + deleteACLCmd(), + ) + RootCmd.AddCommand(deleteCmd) +} + +func deletePreRun(cmd *cobra.Command, args []string) error { + return deleteConfig.shared.validate() +} + +func deleteACLCmd() *cobra.Command { + return &cobra.Command{ + Use: "acl [flags]", + Short: "Delete an ACL", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := deleteConfig.shared.getAdminClient(ctx, sess, false) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + + filter := kafka.ACLFilter{} + return cliRunner.DeleteACL(ctx, filter) + }, + } +} diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 0d5b2429..5df6a706 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -843,6 +843,28 @@ func (c *BrokerAdminClient) CreateACLs( return nil } +// DeleteACLs deletes ACLs in the cluster. +func (c *BrokerAdminClient) DeleteACLs( + ctx context.Context, + filters []kafka.DeleteACLsFilter, +) (*kafka.DeleteACLsResponse, error) { + if c.config.ReadOnly { + return nil, errors.New("Cannot delete ACL in read-only mode") + } + + req := kafka.DeleteACLsRequest{ + Filters: filters, + } + log.Debugf("DeleteACLs request: %+v", req) + + resp, err := c.client.DeleteACLs(ctx, &req) + log.Debugf("DeleteACLs response: %+v (%+v)", resp, err) + if err != nil { + return nil, err + } + return resp, nil +} + func (c *BrokerAdminClient) GetAllTopicsMetadata( ctx context.Context, ) (*kafka.MetadataResponse, error) { diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 542d225c..ddca79ee 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -664,6 +664,102 @@ func TestBrokerClientCreateGetACL(t *testing.T) { assert.Equal(t, expected, aclsInfo) } +func TestBrokerClientDeleteACL(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, + ) + require.NoError(t, err) + + principal := util.RandomString("User:user-create-", 6) + topicName := util.RandomString("topic-create-", 6) + + err = client.CreateACLs( + ctx, + []kafka.ACLEntry{ + { + Principal: principal, + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + ResourceType: kafka.ResourceTypeTopic, + ResourcePatternType: kafka.PatternTypeLiteral, + ResourceName: topicName, + Host: "*", + }, + }, + ) + require.NoError(t, err) + + filter := kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + } + + aclsInfo, err := client.GetACLs(ctx, filter) + require.NoError(t, err) + expected := []ACLInfo{ + { + ResourceType: ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + } + assert.Equal(t, expected, aclsInfo) + + deleteFilters := []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + }, + } + + resp, err := client.DeleteACLs(ctx, deleteFilters) + require.NoError(t, err) + expectedDeleteResults := []kafka.DeleteACLsResult{ + { + Error: nil, + MatchingACLs: []kafka.DeleteACLsMatchingACLs{ + { + Error: nil, + ResourceType: kafka.ResourceTypeTopic, + ResourceName: topicName, + ResourcePatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + }, + }, + }, + } + + require.NoError(t, err) + assert.Equal(t, expectedDeleteResults, resp.Results) + + aclsInfo, err = client.GetACLs(ctx, filter) + require.NoError(t, err) + assert.Equal(t, []ACLInfo{}, aclsInfo) +} + func TestBrokerClientCreateGetUsers(t *testing.T) { if !util.CanTestBrokerAdminSecurity() { t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 93d20e00..964292e4 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -83,6 +83,12 @@ type Client interface { acls []kafka.ACLEntry, ) error + // DeleteACLs deletes ACLs in the cluster. + DeleteACLs( + ctx context.Context, + filters []kafka.DeleteACLsFilter, + ) (*kafka.DeleteACLsResponse, error) + // UpsertUser creates or updates an user in zookeeper. UpsertUser( ctx context.Context, diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 8f7f4d9d..6d18e881 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -436,6 +436,13 @@ func (c *ZKAdminClient) CreateACLs( return errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } +func (c *ZKAdminClient) DeleteACLs( + ctx context.Context, + filters []kafka.DeleteACLsFilter, +) (*kafka.DeleteACLsResponse, error) { + return nil, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") +} + func (c *ZKAdminClient) GetUsers( ctx context.Context, names []string, diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 72ec4c10..3301a737 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1107,6 +1107,21 @@ func TestZkCreateACL(t *testing.T) { assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } +func TestZkDeleteACL(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + _, err = adminClient.DeleteACLs(ctx, []kafka.DeleteACLsFilter{}) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) +} + func TestZkGetUsers(t *testing.T) { ctx := context.Background() adminClient, err := NewZKAdminClient( From 4692583c1486d7ba271139691150ff437e982b46 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 21 Nov 2023 12:16:28 -0500 Subject: [PATCH 101/116] integrate cli and add docs --- README.md | 12 +++++++ cmd/topicctl/subcmd/delete.go | 66 +++++++++++++++++++++++++++++++++-- pkg/cli/cli.go | 54 ++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7a408a3f..09da81fd 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,18 @@ The `create` command creates resources in the cluster from a configuration file. Currently, only ACLs are supported. The create command is separate from the apply command as it is intended for usage with immutable resources managed by topicctl. +#### delete + +``` +topicctl delete [flags] [operation] +``` + +The `delete` subcommand deletes a particular resource type in the cluster. +Currently, the following operations are supported: +| Subcommand | Description | +| --------- | ----------- | +| `delete acl [flags]` | Deletes a single ACL in the cluster matching the provided flags | + #### get ``` diff --git a/cmd/topicctl/subcmd/delete.go b/cmd/topicctl/subcmd/delete.go index eb92e605..5e1ccc0f 100644 --- a/cmd/topicctl/subcmd/delete.go +++ b/cmd/topicctl/subcmd/delete.go @@ -41,8 +41,10 @@ func deletePreRun(cmd *cobra.Command, args []string) error { return deleteConfig.shared.validate() } +var deleteACLsConfig = aclsCmdConfig{} + func deleteACLCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "acl [flags]", Short: "Delete an ACL", Args: cobra.NoArgs, @@ -58,8 +60,68 @@ func deleteACLCmd() *cobra.Command { cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) - filter := kafka.ACLFilter{} + filter := kafka.DeleteACLsFilter{ + ResourceTypeFilter: kafka.ResourceType(deleteACLsConfig.resourceType), + ResourceNameFilter: deleteACLsConfig.resourceNameFilter, + ResourcePatternTypeFilter: kafka.PatternType(deleteACLsConfig.resourcePatternType), + PrincipalFilter: deleteACLsConfig.principalFilter, + HostFilter: deleteACLsConfig.hostFilter, + Operation: kafka.ACLOperationType(deleteACLsConfig.operationType), + PermissionType: kafka.ACLPermissionType(deleteACLsConfig.permissionType), + } + return cliRunner.DeleteACL(ctx, filter) }, } + cmd.Flags().StringVar( + &deleteACLsConfig.hostFilter, + "host", + "", + `The host to filter on. (e.g. 198.51.100.0) (Required)`, + ) + cmd.MarkFlagRequired("host") + + cmd.Flags().Var( + &deleteACLsConfig.operationType, + "operation", + `The operation that is being allowed or denied to filter on. allowed: [any, all, read, write, create, delete, alter, describe, clusteraction, describeconfigs, alterconfigs, idempotentwrite] (Required)`, + ) + cmd.MarkFlagRequired("operation") + + cmd.Flags().Var( + &deleteACLsConfig.permissionType, + "permission-type", + `The permission type to filter on. allowed: [any, allow, deny] (Required)`, + ) + cmd.MarkFlagRequired("permission-type") + + cmd.Flags().StringVar( + &deleteACLsConfig.principalFilter, + "principal", + "", + `The principal to filter on in principalType:name format (e.g. User:alice). (Required)`, + ) + cmd.MarkFlagRequired("principal") + + cmd.Flags().StringVar( + &deleteACLsConfig.resourceNameFilter, + "resource-name", + "", + `The resource name to filter on. (e.g. my-topic) (Required)`, + ) + cmd.MarkFlagRequired("resource-name") + + cmd.Flags().Var( + &deleteACLsConfig.resourcePatternType, + "resource-pattern-type", + `The type of the resource pattern or filter. allowed: [any, match, literal, prefixed]. "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, + ) + + cmd.Flags().Var( + &deleteACLsConfig.resourceType, + "resource-type", + `The type of resource to filter on. allowed: [any, topic, group, cluster, transactionalid, delegationtoken] (Required)`, + ) + cmd.MarkFlagRequired("resource-type") + return cmd } diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 898f4f45..58056a8d 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -21,6 +21,7 @@ import ( "github.com/segmentio/topicctl/pkg/create" "github.com/segmentio/topicctl/pkg/groups" "github.com/segmentio/topicctl/pkg/messages" + "github.com/segmentio/topicctl/pkg/util" log "github.com/sirupsen/logrus" ) @@ -143,6 +144,59 @@ func (c *CLIRunner) CreateACL( return nil } +// DeleteACL deletes a single ACL. +func (c *CLIRunner) DeleteACL(ctx context.Context, filter kafka.DeleteACLsFilter) error { + c.printer("Checking if ACL %v exists...", filter) + c.startSpinner() + // First check that ACL exists + getFilter := kafka.ACLFilter{ + ResourceTypeFilter: filter.ResourceTypeFilter, + ResourceNameFilter: filter.ResourceNameFilter, + ResourcePatternTypeFilter: filter.ResourcePatternTypeFilter, + PrincipalFilter: filter.PrincipalFilter, + HostFilter: filter.HostFilter, + Operation: filter.Operation, + PermissionType: filter.PermissionType, + } + clusterACLs, err := c.adminClient.GetACLs(ctx, getFilter) + c.stopSpinner() + if err != nil { + return fmt.Errorf("Error fetching ACL info: %+v", err) + } + + if len(clusterACLs) == 0 { + return fmt.Errorf("ACL %v does not exist", filter) + } + + if len(clusterACLs) > 1 { + return fmt.Errorf("Delete filter should only match a single ACL, got: %v ", clusterACLs) + } + + clusterACL := clusterACLs[0] + + c.printer("ACL %v exists in the cluster!", clusterACL) + + confirm, err := util.Confirm(fmt.Sprintf("Delete ACL \"%v\"", clusterACL), false) + if err != nil { + return err + } + + if !confirm { + return nil + } + + c.startSpinner() + _, err = c.adminClient.DeleteACLs(ctx, []kafka.DeleteACLsFilter{filter}) + c.stopSpinner() + if err != nil { + return err + } + + c.printer("ACL %v successfully deleted", filter) + + return nil +} + // BootstrapTopics creates configs for one or more topics based on their current state in the // cluster. func (c *CLIRunner) BootstrapTopics( From 434d6028904efb26016df45c542de682b14f168f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 21 Nov 2023 14:36:24 -0500 Subject: [PATCH 102/116] improve formatting --- pkg/admin/types.go | 30 ++++++++++++++++++++++++++ pkg/cli/cli.go | 53 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/pkg/admin/types.go b/pkg/admin/types.go index d55a0e85..b7365adf 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -1,6 +1,7 @@ package admin import ( + "encoding/json" "errors" "fmt" "reflect" @@ -88,6 +89,35 @@ type ACLInfo struct { PermissionType ACLPermissionType `json:"permissionType"` } +// String is used both by fmt.Print and by Cobra in help text. +func FormatACLInfo(a ACLInfo) string { + alias := struct { + ResourceType string + ResourceName string + PatternType string + Principal string + Host string + Operation string + PermissionType string + }{ + ResourceType: a.ResourceType.String(), + ResourceName: a.ResourceName, + PatternType: a.PatternType.String(), + Principal: a.Principal, + Host: a.Host, + Operation: a.Operation.String(), + PermissionType: a.PermissionType.String(), + } + + content, err := json.MarshalIndent(alias, "", " ") + if err != nil { + log.Warnf("Error marshalling acls: %+v", err) + return "Error" + } + + return string(content) +} + // ResourceType presents the Kafka resource type. // We need to subtype this to be able to define methods to // satisfy the Value interface from Cobra so we can use it diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 58056a8d..4dd5fc32 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -2,6 +2,8 @@ package cli import ( "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -144,9 +146,19 @@ func (c *CLIRunner) CreateACL( return nil } +func formatACLs(acls interface{}) string { + content, err := json.MarshalIndent(acls, "", " ") + if err != nil { + log.Warnf("Error marshalling acls: %+v", err) + return "Error" + } + + return string(content) +} + // DeleteACL deletes a single ACL. func (c *CLIRunner) DeleteACL(ctx context.Context, filter kafka.DeleteACLsFilter) error { - c.printer("Checking if ACL %v exists...", filter) + c.printer("Checking if ACL exists for filter:\n%+v", formatACLs(filter)) c.startSpinner() // First check that ACL exists getFilter := kafka.ACLFilter{ @@ -161,38 +173,61 @@ func (c *CLIRunner) DeleteACL(ctx context.Context, filter kafka.DeleteACLsFilter clusterACLs, err := c.adminClient.GetACLs(ctx, getFilter) c.stopSpinner() if err != nil { - return fmt.Errorf("Error fetching ACL info: %+v", err) + return fmt.Errorf("Error fetching ACL info: \n%+v", err) } if len(clusterACLs) == 0 { - return fmt.Errorf("ACL %v does not exist", filter) + return fmt.Errorf("No ACL matches filter:\n%+v", formatACLs(filter)) } if len(clusterACLs) > 1 { - return fmt.Errorf("Delete filter should only match a single ACL, got: %v ", clusterACLs) + return fmt.Errorf("Delete filter should only match a single ACL, got: \n%+v ", formatACLs(clusterACLs)) } clusterACL := clusterACLs[0] - c.printer("ACL %v exists in the cluster!", clusterACL) + c.printer("ACL exists in the cluster:\n%+v", admin.FormatACLInfo(clusterACL)) - confirm, err := util.Confirm(fmt.Sprintf("Delete ACL \"%v\"", clusterACL), false) + confirm, err := util.Confirm("Delete ACL?", false) if err != nil { return err } if !confirm { - return nil + return errors.New("Stopping because of user response") } c.startSpinner() - _, err = c.adminClient.DeleteACLs(ctx, []kafka.DeleteACLsFilter{filter}) + resp, err := c.adminClient.DeleteACLs(ctx, []kafka.DeleteACLsFilter{filter}) c.stopSpinner() if err != nil { return err } - c.printer("ACL %v successfully deleted", filter) + var respErrors = []error{} + var deletedACLs = []kafka.DeleteACLsMatchingACLs{} + + for _, result := range resp.Results { + if result.Error != nil { + respErrors = append(respErrors, result.Error) + } + for _, matchingACL := range result.MatchingACLs { + if matchingACL.Error != nil { + respErrors = append(respErrors, result.Error) + } + deletedACLs = append(deletedACLs, matchingACL) + } + } + + if len(respErrors) > 0 { + return fmt.Errorf("Got errors while deleting ACLs: \n%+v", respErrors) + } + + if len(deletedACLs) != 1 { + return fmt.Errorf("Expected to delete one ACL, got: \n%+v", deletedACLs) + } + + c.printer("ACL successfully deleted: %+v", formatACLs(deletedACLs[0])) return nil } From 2f593faee0593b5f916f8500d2705304cfc56cf3 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 21 Nov 2023 14:41:38 -0500 Subject: [PATCH 103/116] add read only test --- pkg/admin/brokerclient_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index ddca79ee..763110b6 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -868,3 +868,24 @@ func TestBrokerClientCreateACLReadOnly(t *testing.T) { assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) } + +func TestBrokerClientDeleteACLReadOnly(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ReadOnly: true, + }, + ) + require.NoError(t, err) + _, err = client.DeleteACLs(ctx, []kafka.DeleteACLsFilter{}) + + assert.Equal(t, errors.New("Cannot delete ACL in read-only mode"), err) +} From cc4eb74517491a8748176edd7805615d53692d33 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 21 Nov 2023 14:46:46 -0500 Subject: [PATCH 104/116] improve documentation --- cmd/topicctl/subcmd/delete.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/delete.go b/cmd/topicctl/subcmd/delete.go index 5e1ccc0f..d49ceec5 100644 --- a/cmd/topicctl/subcmd/delete.go +++ b/cmd/topicctl/subcmd/delete.go @@ -46,8 +46,11 @@ var deleteACLsConfig = aclsCmdConfig{} func deleteACLCmd() *cobra.Command { cmd := &cobra.Command{ Use: "acl [flags]", - Short: "Delete an ACL", + Short: "Delete an ACL. Requires providing flags to only target a single ACL for deletion.", Args: cobra.NoArgs, + Example: `Delete read acls for topic my-topic, user 'User:default', and host '*' +$ topicctl delete acl --resource-type topic --resource-pattern-type literal --resource-name my-topic --principal 'User:default' --host '*' --operation read --permission-type allow +`, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() sess := session.Must(session.NewSession()) From f0c3f09c0ce674dc069976713fbdffe371934a5b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 22 Nov 2023 11:23:30 -0500 Subject: [PATCH 105/116] fix docstring and error message --- pkg/admin/types.go | 3 ++- pkg/cli/cli.go | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/admin/types.go b/pkg/admin/types.go index b7365adf..9d25b85b 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -89,7 +89,8 @@ type ACLInfo struct { PermissionType ACLPermissionType `json:"permissionType"` } -// String is used both by fmt.Print and by Cobra in help text. +// FormatACLInfo formats an ACLInfo struct as a string, using the +// string version of all the fields. func FormatACLInfo(a ACLInfo) string { alias := struct { ResourceType string diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 4dd5fc32..eae432cb 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -181,7 +181,11 @@ func (c *CLIRunner) DeleteACL(ctx context.Context, filter kafka.DeleteACLsFilter } if len(clusterACLs) > 1 { - return fmt.Errorf("Delete filter should only match a single ACL, got: \n%+v ", formatACLs(clusterACLs)) + var formattedClusterACLs []string + for _, clusterACL := range clusterACLs { + formattedClusterACLs = append(formattedClusterACLs, admin.FormatACLInfo(clusterACL)) + } + return fmt.Errorf("Delete filter should only match a single ACL. Use more specific filter flags to narrow down on a single ACL. ACLs matching filter: \n%+v", strings.Join(formattedClusterACLs, "\n")) } clusterACL := clusterACLs[0] From bd6018425b5fcc4129ff85a4714d07097349e951 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 27 Nov 2023 14:14:05 -0500 Subject: [PATCH 106/116] move things into new acl package and start on dry run --- cmd/topicctl/subcmd/create.go | 6 ++--- cmd/topicctl/subcmd/delete.go | 11 +++++++- pkg/{create => acl}/acl.go | 28 ++++++++++--------- pkg/{create => acl}/acl_test.go | 48 ++++++++++++++++----------------- pkg/cli/cli.go | 16 +++++------ 5 files changed, 60 insertions(+), 49 deletions(-) rename pkg/{create => acl}/acl.go (83%) rename pkg/{create => acl}/acl_test.go (89%) diff --git a/cmd/topicctl/subcmd/create.go b/cmd/topicctl/subcmd/create.go index 09a1299b..7bef8883 100644 --- a/cmd/topicctl/subcmd/create.go +++ b/cmd/topicctl/subcmd/create.go @@ -8,10 +8,10 @@ import ( "path/filepath" "syscall" + "github.com/segmentio/topicctl/pkg/acl" "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" "github.com/segmentio/topicctl/pkg/config" - "github.com/segmentio/topicctl/pkg/create" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -171,14 +171,14 @@ func createACL( clusterConfigPath, ) - creatorConfig := create.ACLCreatorConfig{ + aclAdminConfig := acl.ACLAdminConfig{ DryRun: createConfig.dryRun, SkipConfirm: createConfig.skipConfirm, ACLConfig: aclConfig, ClusterConfig: clusterConfig, } - if err := cliRunner.CreateACL(ctx, creatorConfig); err != nil { + if err := cliRunner.CreateACL(ctx, aclAdminConfig); err != nil { return err } } diff --git a/cmd/topicctl/subcmd/delete.go b/cmd/topicctl/subcmd/delete.go index d49ceec5..78d328ea 100644 --- a/cmd/topicctl/subcmd/delete.go +++ b/cmd/topicctl/subcmd/delete.go @@ -24,12 +24,21 @@ var deleteCmd = &cobra.Command{ } type deleteCmdConfig struct { + dryRun bool + shared sharedOptions } var deleteConfig deleteCmdConfig func init() { + deleteCmd.PersistentFlags().BoolVar( + &deleteConfig.dryRun, + "dry-run", + false, + "Do a dry-run", + ) + addSharedFlags(deleteCmd, &deleteConfig.shared) deleteCmd.AddCommand( deleteACLCmd(), @@ -55,7 +64,7 @@ $ topicctl delete acl --resource-type topic --resource-pattern-type literal --re ctx := context.Background() sess := session.Must(session.NewSession()) - adminClient, err := deleteConfig.shared.getAdminClient(ctx, sess, false) + adminClient, err := deleteConfig.shared.getAdminClient(ctx, sess, deleteConfig.dryRun) if err != nil { return err } diff --git a/pkg/create/acl.go b/pkg/acl/acl.go similarity index 83% rename from pkg/create/acl.go rename to pkg/acl/acl.go index 882524c7..33a87903 100644 --- a/pkg/create/acl.go +++ b/pkg/acl/acl.go @@ -1,4 +1,4 @@ -package create +package acl import ( "context" @@ -13,40 +13,42 @@ import ( log "github.com/sirupsen/logrus" ) -// ACLCreatorConfig contains the configuration for an ACL creator. -type ACLCreatorConfig struct { +// ACLCreatorConfig contains the configuration for an ACL admin. +type ACLAdminConfig struct { ClusterConfig config.ClusterConfig DryRun bool SkipConfirm bool ACLConfig config.ACLConfig } -type ACLCreator struct { - config ACLCreatorConfig +// ACLAdmin executes operations on ACLs by comparing the current ACLs with the desired ACLs. +type ACLAdmin struct { + config ACLAdminConfig adminClient admin.Client clusterConfig config.ClusterConfig aclConfig config.ACLConfig } -func NewACLCreator( +func NewACLAdmin( ctx context.Context, adminClient admin.Client, - creatorConfig ACLCreatorConfig, -) (*ACLCreator, error) { + aclAdminConfig ACLAdminConfig, +) (*ACLAdmin, error) { if !adminClient.GetSupportedFeatures().ACLs { return nil, fmt.Errorf("ACLs are not supported by this cluster") } - return &ACLCreator{ - config: creatorConfig, + return &ACLAdmin{ + config: aclAdminConfig, adminClient: adminClient, - clusterConfig: creatorConfig.ClusterConfig, - aclConfig: creatorConfig.ACLConfig, + clusterConfig: aclAdminConfig.ClusterConfig, + aclConfig: aclAdminConfig.ACLConfig, }, nil } -func (a *ACLCreator) Create(ctx context.Context) error { +// Create creates ACLs that do not already exist based on the ACL config. +func (a *ACLAdmin) Create(ctx context.Context) error { log.Info("Validating configs...") if err := a.clusterConfig.Validate(); err != nil { diff --git a/pkg/create/acl_test.go b/pkg/acl/acl_test.go similarity index 89% rename from pkg/create/acl_test.go rename to pkg/acl/acl_test.go index 0797fb81..fa16496e 100644 --- a/pkg/create/acl_test.go +++ b/pkg/acl/acl_test.go @@ -1,4 +1,4 @@ -package create +package acl import ( "context" @@ -49,11 +49,11 @@ func TestCreateNewACLs(t *testing.T) { }, }, } - creator := testCreator(ctx, t, aclConfig) - defer creator.adminClient.Close() + aclAdmin := testACLAdmin(ctx, t, aclConfig) + defer aclAdmin.adminClient.Close() defer func() { - _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + _, err := aclAdmin.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, &kafka.DeleteACLsRequest{ Filters: []kafka.DeleteACLsFilter{ { @@ -73,9 +73,9 @@ func TestCreateNewACLs(t *testing.T) { t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) } }() - err := creator.Create(ctx) + err := aclAdmin.Create(ctx) require.NoError(t, err) - acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + acl, err := aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceTypeTopic, ResourceNameFilter: topicName, ResourcePatternTypeFilter: kafka.PatternTypeLiteral, @@ -134,11 +134,11 @@ func TestCreateExistingACLs(t *testing.T) { }, }, } - creator := testCreator(ctx, t, aclConfig) - defer creator.adminClient.Close() + aclAdmin := testACLAdmin(ctx, t, aclConfig) + defer aclAdmin.adminClient.Close() defer func() { - _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + _, err := aclAdmin.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, &kafka.DeleteACLsRequest{ Filters: []kafka.DeleteACLsFilter{ { @@ -158,9 +158,9 @@ func TestCreateExistingACLs(t *testing.T) { t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) } }() - err := creator.Create(ctx) + err := aclAdmin.Create(ctx) require.NoError(t, err) - acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + acl, err := aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceTypeTopic, ResourceNameFilter: topicName, ResourcePatternTypeFilter: kafka.PatternTypeLiteral, @@ -182,9 +182,9 @@ func TestCreateExistingACLs(t *testing.T) { }, }, acl) // Run create again and make sure it is idempotent - err = creator.Create(ctx) + err = aclAdmin.Create(ctx) require.NoError(t, err) - acl, err = creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + acl, err = aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceTypeTopic, ResourceNameFilter: topicName, ResourcePatternTypeFilter: kafka.PatternTypeLiteral, @@ -243,12 +243,12 @@ func TestCreateACLsDryRun(t *testing.T) { }, }, } - creator := testCreator(ctx, t, aclConfig) - defer creator.adminClient.Close() - creator.config.DryRun = true + aclAdmin := testACLAdmin(ctx, t, aclConfig) + defer aclAdmin.adminClient.Close() + aclAdmin.config.DryRun = true defer func() { - _, err := creator.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + _, err := aclAdmin.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, &kafka.DeleteACLsRequest{ Filters: []kafka.DeleteACLsFilter{ { @@ -268,9 +268,9 @@ func TestCreateACLsDryRun(t *testing.T) { t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) } }() - err := creator.Create(ctx) + err := aclAdmin.Create(ctx) require.NoError(t, err) - acl, err := creator.adminClient.GetACLs(ctx, kafka.ACLFilter{ + acl, err := aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceTypeTopic, ResourceNameFilter: topicName, ResourcePatternTypeFilter: kafka.PatternTypeLiteral, @@ -283,11 +283,11 @@ func TestCreateACLsDryRun(t *testing.T) { require.Equal(t, []admin.ACLInfo{}, acl) } -func testCreator( +func testACLAdmin( ctx context.Context, t *testing.T, aclConfig config.ACLConfig, -) *ACLCreator { +) *ACLAdmin { clusterConfig := config.ClusterConfig{ Meta: config.ClusterMeta{ Name: "test-cluster", @@ -303,10 +303,10 @@ func testCreator( adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") require.NoError(t, err) - applier, err := NewACLCreator( + aclAdmin, err := NewACLAdmin( ctx, adminClient, - ACLCreatorConfig{ + ACLAdminConfig{ ClusterConfig: clusterConfig, ACLConfig: aclConfig, DryRun: false, @@ -314,5 +314,5 @@ func testCreator( }, ) require.NoError(t, err) - return applier + return aclAdmin } diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index eae432cb..98a0f892 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -16,11 +16,11 @@ import ( "github.com/briandowns/spinner" "github.com/fatih/color" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/acl" "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/check" "github.com/segmentio/topicctl/pkg/config" - "github.com/segmentio/topicctl/pkg/create" "github.com/segmentio/topicctl/pkg/groups" "github.com/segmentio/topicctl/pkg/messages" "github.com/segmentio/topicctl/pkg/util" @@ -117,12 +117,12 @@ func (c *CLIRunner) ApplyTopic( // CreateACL does an apply run according to the spec in the argument config. func (c *CLIRunner) CreateACL( ctx context.Context, - creatorConfig create.ACLCreatorConfig, + aclAdminConfig acl.ACLAdminConfig, ) error { - creator, err := create.NewACLCreator( + aclAdmin, err := acl.NewACLAdmin( ctx, c.adminClient, - creatorConfig, + aclAdminConfig, ) if err != nil { return err @@ -132,12 +132,12 @@ func (c *CLIRunner) CreateACL( c.printer( "Starting creation for ACLs %s in environment %s, cluster %s", - highlighter(creatorConfig.ACLConfig.Meta.Name), - highlighter(creatorConfig.ACLConfig.Meta.Environment), - highlighter(creatorConfig.ACLConfig.Meta.Cluster), + highlighter(aclAdminConfig.ACLConfig.Meta.Name), + highlighter(aclAdminConfig.ACLConfig.Meta.Environment), + highlighter(aclAdminConfig.ACLConfig.Meta.Cluster), ) - err = creator.Create(ctx) + err = aclAdmin.Create(ctx) if err != nil { return err } From 6f9ba6d584156f1e39270819849a3930d22d1ed2 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 1 Dec 2023 15:20:56 -0500 Subject: [PATCH 107/116] finish dry run --- cmd/topicctl/subcmd/delete.go | 10 +- pkg/acl/acl.go | 95 +++++++++++++++++ pkg/acl/acl_test.go | 187 +++++++++++++++++++++++++++++++++- pkg/cli/cli.go | 99 ++++-------------- 4 files changed, 309 insertions(+), 82 deletions(-) diff --git a/cmd/topicctl/subcmd/delete.go b/cmd/topicctl/subcmd/delete.go index 78d328ea..a009ae94 100644 --- a/cmd/topicctl/subcmd/delete.go +++ b/cmd/topicctl/subcmd/delete.go @@ -6,6 +6,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/acl" "github.com/segmentio/topicctl/pkg/cli" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -82,7 +83,14 @@ $ topicctl delete acl --resource-type topic --resource-pattern-type literal --re PermissionType: kafka.ACLPermissionType(deleteACLsConfig.permissionType), } - return cliRunner.DeleteACL(ctx, filter) + aclAdminConfig := acl.ACLAdminConfig{ + // Omit fields we don't need for deletes + DryRun: deleteConfig.dryRun, + // Deletes cannot be skipped + SkipConfirm: false, + } + + return cliRunner.DeleteACL(ctx, aclAdminConfig, filter) }, } cmd.Flags().StringVar( diff --git a/pkg/acl/acl.go b/pkg/acl/acl.go index 33a87903..ff7a131a 100644 --- a/pkg/acl/acl.go +++ b/pkg/acl/acl.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/admin" @@ -141,3 +142,97 @@ func formatNewACLsConfig(config []kafka.ACLEntry) string { return string(content) } + +// Delete checks if ACLs exist and deletes them if they do. +func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) error { + log.Infof("Checking if ACL exists for filter:\n%+v", formatACLs(filter)) + + getFilter := kafka.ACLFilter{ + ResourceTypeFilter: filter.ResourceTypeFilter, + ResourceNameFilter: filter.ResourceNameFilter, + ResourcePatternTypeFilter: filter.ResourcePatternTypeFilter, + PrincipalFilter: filter.PrincipalFilter, + HostFilter: filter.HostFilter, + Operation: filter.Operation, + PermissionType: filter.PermissionType, + } + clusterACLs, err := a.adminClient.GetACLs(ctx, getFilter) + + if err != nil { + return fmt.Errorf("Error fetching ACL info: \n%+v", err) + } + + if len(clusterACLs) == 0 { + return fmt.Errorf("No ACL matches filter:\n%+v", formatACLs(filter)) + } + + if len(clusterACLs) > 1 { + var formattedClusterACLs []string + for _, clusterACL := range clusterACLs { + formattedClusterACLs = append(formattedClusterACLs, admin.FormatACLInfo(clusterACL)) + } + return fmt.Errorf("Delete filter should only match a single ACL. Use more specific filter flags to narrow down on a single ACL. ACLs matching filter: \n%+v", strings.Join(formattedClusterACLs, "\n")) + } + + clusterACL := clusterACLs[0] + + log.Infof("ACL exists in the cluster:\n%+v", admin.FormatACLInfo(clusterACL)) + + if a.config.DryRun { + log.Infof("Would delete ACLs:\n%+v", admin.FormatACLInfo(clusterACL)) + return nil + } + + // This isn't settable by the CLI for safety measures but allows for testability + confirm, err := util.Confirm("Delete ACL?", a.config.SkipConfirm) + if err != nil { + return err + } + + if !confirm { + return errors.New("Stopping because of user response") + } + + resp, err := a.adminClient.DeleteACLs(ctx, []kafka.DeleteACLsFilter{filter}) + + if err != nil { + return err + } + + var respErrors = []error{} + var deletedACLs = []kafka.DeleteACLsMatchingACLs{} + + for _, result := range resp.Results { + if result.Error != nil { + respErrors = append(respErrors, result.Error) + } + for _, matchingACL := range result.MatchingACLs { + if matchingACL.Error != nil { + respErrors = append(respErrors, result.Error) + } + deletedACLs = append(deletedACLs, matchingACL) + } + } + + if len(respErrors) > 0 { + return fmt.Errorf("Got errors while deleting ACLs: \n%+v", respErrors) + } + + if len(deletedACLs) != 1 { + return fmt.Errorf("Expected to delete one ACL, got: \n%+v", deletedACLs) + } + + log.Infof("ACL successfully deleted: %+v", formatACLs(deletedACLs[0])) + + return nil +} + +func formatACLs(acls interface{}) string { + content, err := json.MarshalIndent(acls, "", " ") + if err != nil { + log.Warnf("Error marshalling acls: %+v", err) + return "Error" + } + + return string(content) +} diff --git a/pkg/acl/acl_test.go b/pkg/acl/acl_test.go index fa16496e..ec39c1bf 100644 --- a/pkg/acl/acl_test.go +++ b/pkg/acl/acl_test.go @@ -247,13 +247,153 @@ func TestCreateACLsDryRun(t *testing.T) { defer aclAdmin.adminClient.Close() aclAdmin.config.DryRun = true + err := aclAdmin.Create(ctx) + require.NoError(t, err) + acl, err := aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{}, acl) +} + +func TestDeleteACLs(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-delete-", 6) + topicName := util.RandomString("acl-delete-", 6) + + aclConfig := config.ACLConfig{ + Meta: config.ResourceMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + aclAdmin := testACLAdmin(ctx, t, aclConfig) + defer aclAdmin.adminClient.Close() + + err := aclAdmin.Create(ctx) + require.NoError(t, err) + acl, err := aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) + + err = aclAdmin.Delete(ctx, kafka.DeleteACLsFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + acl, err = aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{}, acl) +} + +func TestDeleteACLDoesNotExist(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-delete-", 6) + topicName := util.RandomString("acl-delete-", 6) + + aclConfig := config.ACLConfig{ + Meta: config.ResourceMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + aclAdmin := testACLAdmin(ctx, t, aclConfig) + defer aclAdmin.adminClient.Close() + defer func() { _, err := aclAdmin.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, &kafka.DeleteACLsRequest{ Filters: []kafka.DeleteACLsFilter{ { ResourceTypeFilter: kafka.ResourceTypeTopic, - ResourceNameFilter: topicName, + ResourceNameFilter: "does-not-exist", ResourcePatternTypeFilter: kafka.PatternTypeLiteral, PrincipalFilter: principal, HostFilter: "*", @@ -280,7 +420,50 @@ func TestCreateACLsDryRun(t *testing.T) { Operation: kafka.ACLOperationTypeRead, }) require.NoError(t, err) - require.Equal(t, []admin.ACLInfo{}, acl) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) + + err = aclAdmin.Delete(ctx, kafka.DeleteACLsFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: "does-not-exist", + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.Error(t, err) + // ACL still exists + acl, err = aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) } func testACLAdmin( diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 98a0f892..c85603e3 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -2,8 +2,6 @@ package cli import ( "context" - "encoding/json" - "errors" "fmt" "os" "path/filepath" @@ -23,7 +21,6 @@ import ( "github.com/segmentio/topicctl/pkg/config" "github.com/segmentio/topicctl/pkg/groups" "github.com/segmentio/topicctl/pkg/messages" - "github.com/segmentio/topicctl/pkg/util" log "github.com/sirupsen/logrus" ) @@ -124,6 +121,7 @@ func (c *CLIRunner) CreateACL( c.adminClient, aclAdminConfig, ) + if err != nil { return err } @@ -146,93 +144,36 @@ func (c *CLIRunner) CreateACL( return nil } -func formatACLs(acls interface{}) string { - content, err := json.MarshalIndent(acls, "", " ") - if err != nil { - log.Warnf("Error marshalling acls: %+v", err) - return "Error" - } - - return string(content) -} - // DeleteACL deletes a single ACL. -func (c *CLIRunner) DeleteACL(ctx context.Context, filter kafka.DeleteACLsFilter) error { - c.printer("Checking if ACL exists for filter:\n%+v", formatACLs(filter)) - c.startSpinner() - // First check that ACL exists - getFilter := kafka.ACLFilter{ - ResourceTypeFilter: filter.ResourceTypeFilter, - ResourceNameFilter: filter.ResourceNameFilter, - ResourcePatternTypeFilter: filter.ResourcePatternTypeFilter, - PrincipalFilter: filter.PrincipalFilter, - HostFilter: filter.HostFilter, - Operation: filter.Operation, - PermissionType: filter.PermissionType, - } - clusterACLs, err := c.adminClient.GetACLs(ctx, getFilter) - c.stopSpinner() - if err != nil { - return fmt.Errorf("Error fetching ACL info: \n%+v", err) - } - - if len(clusterACLs) == 0 { - return fmt.Errorf("No ACL matches filter:\n%+v", formatACLs(filter)) - } - - if len(clusterACLs) > 1 { - var formattedClusterACLs []string - for _, clusterACL := range clusterACLs { - formattedClusterACLs = append(formattedClusterACLs, admin.FormatACLInfo(clusterACL)) - } - return fmt.Errorf("Delete filter should only match a single ACL. Use more specific filter flags to narrow down on a single ACL. ACLs matching filter: \n%+v", strings.Join(formattedClusterACLs, "\n")) - } - - clusterACL := clusterACLs[0] - - c.printer("ACL exists in the cluster:\n%+v", admin.FormatACLInfo(clusterACL)) +func (c *CLIRunner) DeleteACL( + ctx context.Context, + aclAdminConfig acl.ACLAdminConfig, + filter kafka.DeleteACLsFilter, +) error { + aclAdmin, err := acl.NewACLAdmin( + ctx, + c.adminClient, + aclAdminConfig, + ) - confirm, err := util.Confirm("Delete ACL?", false) if err != nil { return err } - if !confirm { - return errors.New("Stopping because of user response") - } + highlighter := color.New(color.FgYellow, color.Bold).SprintfFunc() - c.startSpinner() - resp, err := c.adminClient.DeleteACLs(ctx, []kafka.DeleteACLsFilter{filter}) - c.stopSpinner() + c.printer( + "Starting deletion for ACLs in environment %s, cluster %s", + highlighter(aclAdminConfig.ACLConfig.Meta.Environment), + highlighter(aclAdminConfig.ACLConfig.Meta.Cluster), + ) + + err = aclAdmin.Delete(ctx, filter) if err != nil { return err } - var respErrors = []error{} - var deletedACLs = []kafka.DeleteACLsMatchingACLs{} - - for _, result := range resp.Results { - if result.Error != nil { - respErrors = append(respErrors, result.Error) - } - for _, matchingACL := range result.MatchingACLs { - if matchingACL.Error != nil { - respErrors = append(respErrors, result.Error) - } - deletedACLs = append(deletedACLs, matchingACL) - } - } - - if len(respErrors) > 0 { - return fmt.Errorf("Got errors while deleting ACLs: \n%+v", respErrors) - } - - if len(deletedACLs) != 1 { - return fmt.Errorf("Expected to delete one ACL, got: \n%+v", deletedACLs) - } - - c.printer("ACL successfully deleted: %+v", formatACLs(deletedACLs[0])) - + c.printer("Delete completed successfully!") return nil } From 4545a24fee553507ea682ee4b89323476b288a2c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 1 Dec 2023 15:57:18 -0500 Subject: [PATCH 108/116] support deleting multiple acls --- pkg/acl/acl.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/pkg/acl/acl.go b/pkg/acl/acl.go index ff7a131a..364b0edc 100644 --- a/pkg/acl/acl.go +++ b/pkg/acl/acl.go @@ -166,25 +166,17 @@ func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) er return fmt.Errorf("No ACL matches filter:\n%+v", formatACLs(filter)) } - if len(clusterACLs) > 1 { - var formattedClusterACLs []string - for _, clusterACL := range clusterACLs { - formattedClusterACLs = append(formattedClusterACLs, admin.FormatACLInfo(clusterACL)) - } - return fmt.Errorf("Delete filter should only match a single ACL. Use more specific filter flags to narrow down on a single ACL. ACLs matching filter: \n%+v", strings.Join(formattedClusterACLs, "\n")) - } - - clusterACL := clusterACLs[0] - - log.Infof("ACL exists in the cluster:\n%+v", admin.FormatACLInfo(clusterACL)) + log.Infof("ACLs exists in the cluster:\n%+v", formatACLInfos(clusterACLs)) if a.config.DryRun { - log.Infof("Would delete ACLs:\n%+v", admin.FormatACLInfo(clusterACL)) + log.Infof("Would delete ACLs:\n%+v", formatACLInfos(clusterACLs)) return nil } + // TODO: step through each ACL and prompt for deletion + // This isn't settable by the CLI for safety measures but allows for testability - confirm, err := util.Confirm("Delete ACL?", a.config.SkipConfirm) + confirm, err := util.Confirm("Delete ACLs?", a.config.SkipConfirm) if err != nil { return err } @@ -218,11 +210,7 @@ func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) er return fmt.Errorf("Got errors while deleting ACLs: \n%+v", respErrors) } - if len(deletedACLs) != 1 { - return fmt.Errorf("Expected to delete one ACL, got: \n%+v", deletedACLs) - } - - log.Infof("ACL successfully deleted: %+v", formatACLs(deletedACLs[0])) + log.Infof("ACLs successfully deleted: %+v", formatACLs(deletedACLs)) return nil } @@ -236,3 +224,13 @@ func formatACLs(acls interface{}) string { return string(content) } + +func formatACLInfos(acls []admin.ACLInfo) string { + aclsString := []string{} + + for _, acl := range acls { + aclsString = append(aclsString, admin.FormatACLInfo(acl)) + } + + return strings.Join(aclsString, "\n") +} From a32a02ca59e067c928af27a55a12a6bbb4d07c05 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 1 Dec 2023 16:15:25 -0500 Subject: [PATCH 109/116] add test for multiple deletes --- pkg/acl/acl_test.go | 190 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/pkg/acl/acl_test.go b/pkg/acl/acl_test.go index ec39c1bf..f52594a1 100644 --- a/pkg/acl/acl_test.go +++ b/pkg/acl/acl_test.go @@ -348,6 +348,196 @@ func TestDeleteACLs(t *testing.T) { require.Equal(t, []admin.ACLInfo{}, acl) } +func TestDeleteMultipleACLs(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + principal := util.RandomString("User:acl-delete-multi-", 6) + topicName := util.RandomString("acl-delete-multi-", 6) + + // Create 3 ACLs, two for topics and one for groups + aclConfig := config.ACLConfig{ + Meta: config.ResourceMeta{ + Name: "test-acl", + Cluster: "test-cluster", + Region: "test-region", + Environment: "test-environment", + }, + Spec: config.ACLSpec{ + ACLs: []config.ACL{ + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeTopic, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeDeny, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + { + Resource: config.ACLResource{ + Type: kafka.ResourceTypeGroup, + Name: topicName, + PatternType: kafka.PatternTypeLiteral, + Principal: principal, + Host: "*", + Permission: kafka.ACLPermissionTypeAllow, + }, + Operations: []kafka.ACLOperationType{ + kafka.ACLOperationTypeRead, + }, + }, + }, + }, + } + aclAdmin := testACLAdmin(ctx, t, aclConfig) + defer aclAdmin.adminClient.Close() + + err := aclAdmin.Create(ctx) + require.NoError(t, err) + + defer func() { + _, err := aclAdmin.adminClient.GetConnector().KafkaClient.DeleteACLs(ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeGroup, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + acl, err := aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAny, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + { + ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeDeny), + }, + }, acl) + + acl, err = aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeGroup, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeGroup), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) + + // Delete the two topic ACLs + err = aclAdmin.Delete(ctx, kafka.DeleteACLsFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeAny, + }) + require.NoError(t, err) + acl, err = aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeAny, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{}, acl) + + // Verify the group ACL remains + acl, err = aclAdmin.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeGroup, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + PrincipalFilter: principal, + HostFilter: "*", + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + }) + require.NoError(t, err) + require.Equal(t, []admin.ACLInfo{ + { + ResourceType: admin.ResourceType(kafka.ResourceTypeGroup), + ResourceName: topicName, + PatternType: admin.PatternType(kafka.PatternTypeLiteral), + Principal: principal, + Host: "*", + Operation: admin.ACLOperationType(kafka.ACLOperationTypeRead), + PermissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAllow), + }, + }, acl) +} + func TestDeleteACLDoesNotExist(t *testing.T) { if !util.CanTestBrokerAdminSecurity() { t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") From 42f0d2ffa52de12cb1443716306f887bd04ceb0d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Dec 2023 13:04:37 -0500 Subject: [PATCH 110/116] allow deleting multiple acls --- cmd/topicctl/subcmd/delete.go | 6 +++--- pkg/acl/acl.go | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/topicctl/subcmd/delete.go b/cmd/topicctl/subcmd/delete.go index a009ae94..91f6345d 100644 --- a/cmd/topicctl/subcmd/delete.go +++ b/cmd/topicctl/subcmd/delete.go @@ -55,11 +55,11 @@ var deleteACLsConfig = aclsCmdConfig{} func deleteACLCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "acl [flags]", - Short: "Delete an ACL. Requires providing flags to only target a single ACL for deletion.", + Use: "acls [flags]", + Short: "Delete ACLs. Requires providing flags to target ACLs for deletion.", Args: cobra.NoArgs, Example: `Delete read acls for topic my-topic, user 'User:default', and host '*' -$ topicctl delete acl --resource-type topic --resource-pattern-type literal --resource-name my-topic --principal 'User:default' --host '*' --operation read --permission-type allow +$ topicctl delete acls --resource-type topic --resource-pattern-type literal --resource-name my-topic --principal 'User:default' --host '*' --operation read --permission-type allow `, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() diff --git a/pkg/acl/acl.go b/pkg/acl/acl.go index 364b0edc..5ee6e73a 100644 --- a/pkg/acl/acl.go +++ b/pkg/acl/acl.go @@ -145,7 +145,7 @@ func formatNewACLsConfig(config []kafka.ACLEntry) string { // Delete checks if ACLs exist and deletes them if they do. func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) error { - log.Infof("Checking if ACL exists for filter:\n%+v", formatACLs(filter)) + log.Infof("Checking if ACLs exists for filter:\n%+v", formatACLs(filter)) getFilter := kafka.ACLFilter{ ResourceTypeFilter: filter.ResourceTypeFilter, @@ -166,17 +166,15 @@ func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) er return fmt.Errorf("No ACL matches filter:\n%+v", formatACLs(filter)) } - log.Infof("ACLs exists in the cluster:\n%+v", formatACLInfos(clusterACLs)) + log.Infof("%d ACLs in the cluster match the filter provided", len(clusterACLs)) if a.config.DryRun { log.Infof("Would delete ACLs:\n%+v", formatACLInfos(clusterACLs)) return nil } - // TODO: step through each ACL and prompt for deletion - // This isn't settable by the CLI for safety measures but allows for testability - confirm, err := util.Confirm("Delete ACLs?", a.config.SkipConfirm) + confirm, err := util.Confirm(fmt.Sprintf("Delete ACLs?\n%+v", formatACLInfos(clusterACLs)), a.config.SkipConfirm) if err != nil { return err } From 95571660029332586fd0696d5fc028bd806e6d13 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Dec 2023 13:07:20 -0500 Subject: [PATCH 111/116] remove starting deletion log --- pkg/cli/cli.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index c85603e3..1be85242 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -162,12 +162,6 @@ func (c *CLIRunner) DeleteACL( highlighter := color.New(color.FgYellow, color.Bold).SprintfFunc() - c.printer( - "Starting deletion for ACLs in environment %s, cluster %s", - highlighter(aclAdminConfig.ACLConfig.Meta.Environment), - highlighter(aclAdminConfig.ACLConfig.Meta.Cluster), - ) - err = aclAdmin.Delete(ctx, filter) if err != nil { return err From ebf1cdc782915437b2d3c978479ff0f5c124b56c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Dec 2023 13:12:35 -0500 Subject: [PATCH 112/116] harden test --- pkg/acl/acl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acl/acl_test.go b/pkg/acl/acl_test.go index f52594a1..b377146f 100644 --- a/pkg/acl/acl_test.go +++ b/pkg/acl/acl_test.go @@ -448,7 +448,7 @@ func TestDeleteMultipleACLs(t *testing.T) { Operation: kafka.ACLOperationTypeRead, }) require.NoError(t, err) - require.Equal(t, []admin.ACLInfo{ + require.ElementsMatch(t, []admin.ACLInfo{ { ResourceType: admin.ResourceType(kafka.ResourceTypeTopic), ResourceName: topicName, From 6843aa657c50bcb1e6dc83085280c2ec3a707cba Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Dec 2023 13:13:18 -0500 Subject: [PATCH 113/116] remove unused highlighter --- pkg/cli/cli.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 1be85242..338a44a1 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -160,8 +160,6 @@ func (c *CLIRunner) DeleteACL( return err } - highlighter := color.New(color.FgYellow, color.Bold).SprintfFunc() - err = aclAdmin.Delete(ctx, filter) if err != nil { return err From aadd135dca3994186d9c55e4417c862a973a5ae6 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Dec 2023 13:35:54 -0500 Subject: [PATCH 114/116] rearrange plan for deletion --- pkg/acl/acl.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acl/acl.go b/pkg/acl/acl.go index 5ee6e73a..fdac3172 100644 --- a/pkg/acl/acl.go +++ b/pkg/acl/acl.go @@ -166,7 +166,7 @@ func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) er return fmt.Errorf("No ACL matches filter:\n%+v", formatACLs(filter)) } - log.Infof("%d ACLs in the cluster match the filter provided", len(clusterACLs)) + log.Infof("The following ACLs in the cluster are planned for deletion:\n%+v", formatACLInfos(clusterACLs)) if a.config.DryRun { log.Infof("Would delete ACLs:\n%+v", formatACLInfos(clusterACLs)) @@ -174,7 +174,7 @@ func (a *ACLAdmin) Delete(ctx context.Context, filter kafka.DeleteACLsFilter) er } // This isn't settable by the CLI for safety measures but allows for testability - confirm, err := util.Confirm(fmt.Sprintf("Delete ACLs?\n%+v", formatACLInfos(clusterACLs)), a.config.SkipConfirm) + confirm, err := util.Confirm("Delete ACLs?", a.config.SkipConfirm) if err != nil { return err } From aa793354881d3f30011de5dc520db981386e3c8f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Dec 2023 13:42:40 -0500 Subject: [PATCH 115/116] fix grammar --- pkg/acl/acl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acl/acl.go b/pkg/acl/acl.go index fdac3172..48f02d76 100644 --- a/pkg/acl/acl.go +++ b/pkg/acl/acl.go @@ -113,7 +113,7 @@ func (a *ACLAdmin) Create(ctx context.Context) error { } log.Infof( - "It looks like these ACLs doesn't already exists. Will create them with this config:\n%s", + "It looks like these ACLs don't already exist. Will create them with this config:\n%s", formatNewACLsConfig(newACLs), ) From d648e2893c203b2c70dc67acbee6fe2779edb925 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Dec 2023 15:53:37 -0500 Subject: [PATCH 116/116] fix merge conflict --- pkg/cli/cli.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index b969bae0..783963b8 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -19,7 +19,6 @@ import ( "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/check" "github.com/segmentio/topicctl/pkg/config" - "github.com/segmentio/topicctl/pkg/create" "github.com/segmentio/topicctl/pkg/groups" "github.com/segmentio/topicctl/pkg/messages" log "github.com/sirupsen/logrus"