Merge pull request #36 from cultureamp/jasono/kafkactl

Start installing various Kafka CLI tools in ca-kafka-local plugin

plugins/ca-kafka-local/README.md

@@ -10,6 +10,10 @@ What it provides:
 - Process Compose job
   - Will print information to the terminal about Kafka, once `kafka-local` is up and running.
   - Allows you to add a `depends_on` clause so other services wait for `kafka-local`.
+- Various Kafka CLI Tools (each with configuration to connect to `kafka-local`)
+  - [kafkactl](https://deviceinsight.github.io/kafkactl/)
+  - [kcat](https://github.com/edenhill/kcat)
+  - [kaf](https://github.com/birdayz/kaf)
 
 ## Usage
 
@@ -47,3 +51,17 @@ Please don't start Kafka within your projects `process-compose.yaml` file. For e
 We don't want to set up circular loops where Hotel launches `devbox services up` and then this launches Hotel again.
 
 For now run Hotel commands in a separate terminal. In future we might provide helpers in Hotel that launch services in your project as well as projects you depend on.
+
+## Creating a topic
+
+You can use one of the CLI tools to create topics your service requires.
+
+For example:
+
+    kafkactl create topic local.mytopic.v1
+
+Or
+
+    kaf topic create local.jasontopic.v3
+
+In future we plan to support creating your topics as defined in [kafka-ops](https://github.com/cultureamp/kafka-ops/).

plugins/ca-kafka-local/bin/kaf

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# The Kaf tool doesn't allow setting the config file path via an environment variable.
+# This script is a workaround to set the config file path via the `--config` argument.
+
+KAF_BINARY="$DEVBOX_PACKAGES_DIR/bin/kaf"
+
+if [[ ! -f "$KAF_BINARY" ]]; then
+    echo "Error: $KAF_BINARY was not found in devbox packages."
+    exit 1
+fi
+
+# Call the kaf binary with the `--config` argument and add all passed arguments
+"$KAF_BINARY" --config "$KAF_CONFIG_PATH" "$@"

plugins/ca-kafka-local/bin/kafka-local-readme

@@ -21,3 +21,8 @@ echo
 echo "== Links =="
 echo "Kafka UI: ${KAFKA_UI_URL}"
 echo "Schema Registry: http://${SCHEMA_REGISTRY_URL}"
+echo
+echo "== Tools =="
+echo "kafkactl - https://deviceinsight.github.io/kafkactl/"
+echo "kcat - https://github.com/edenhill/kcat"
+echo "kaf - https://github.com/birdayz/kaf"

plugins/ca-kafka-local/config/kaf/config

@@ -0,0 +1,10 @@
+current-cluster: local
+clusters:
+  - name: local
+    brokers:
+      - ${KAFKA_BROKERS_SASL}
+    SASL:
+      mechanism: SCRAM-SHA-512
+      username: ${KAFKA_SASL_USER}
+      password: ${KAFKA_SASL_PASSWORD}
+    security-protocol: SASL_PLAINTEXT

plugins/ca-kafka-local/config/kafkactl/config.yaml

@@ -0,0 +1,19 @@
+# See https://github.com/deviceinsight/kafkactl?tab=readme-ov-file#configuration
+contexts:
+  local:
+    brokers:
+      - ${KAFKA_BROKERS_SASL}
+
+    tls:
+      enabled: false
+
+    sasl:
+      enabled: true
+      username: ${KAFKA_SASL_USER}
+      password: ${KAFKA_SASL_PASSWORD}
+      mechanism: scram-sha512
+
+    avro:
+      schemaRegistry: ${SCHEMA_REGISTRY_URL}
+
+current-context: local

plugins/ca-kafka-local/config/kcat/kcat.conf

@@ -0,0 +1,5 @@
+bootstrap.servers=${KAFKA_BROKERS_SASL}
+security.protocol=SASL_PLAINTEXT
+sasl.mechanisms=SCRAM-SHA-512
+sasl.username=${KAFKA_SASL_USER}
+sasl.password=${KAFKA_SASL_PASSWORD}

plugins/ca-kafka-local/plugin.json

@@ -3,21 +3,35 @@
   "version": "0.0.1",
   "description": "Environment variables and tools to interact with `kafka-local` in your service",
   "readme": "README.md",
+  "packages": ["kafkactl@5", "kcat@1", "envsubst", "kaf"],
   "env": {
+    "PATH": "{{ .Virtenv }}/bin:$PATH",
     "KAFKA_BROKERS": "localhost:14231",
     "KAFKA_BROKERS_SASL": "localhost:14233",
     "KAFKA_SASL_PORT": "14233",
     "KAFKA_SASL_USER": "user",
     "KAFKA_SASL_PASSWORD": "user-secret",
     "SCHEMA_REGISTRY_URL": "localhost:14228",
     "KAFKA_TOPIC_PREFIX": "local",
-    "KAFKA_UI_URL": "http://localhost:12288"
+    "KAFKA_UI_URL": "http://localhost:12288",
+    "KAFKA_CTL_CONFIG": "{{.Virtenv}}/config/kafkactl/config.yaml",
+    "KCAT_CONFIG": "{{.Virtenv}}/config/kcat/kcat.conf",
+    "KAF_CONFIG_PATH": "{{.Virtenv}}/config/kaf/config"
   },
   "create_files": {
     "{{.Virtenv}}/bin/kafka-local-readme": "bin/kafka-local-readme",
-    "{{.Virtenv}}/config/process-compose.yaml": "config/process-compose.yaml"
+    "{{.Virtenv}}/bin/kaf": "bin/kaf",
+    "{{.Virtenv}}/config/process-compose.yaml": "config/process-compose.yaml",
+    "{{.Virtenv}}/config/kafkactl/config.yaml.template": "config/kafkactl/config.yaml",
+    "{{.Virtenv}}/config/kcat/kcat.conf.template": "config/kcat/kcat.conf",
+    "{{.Virtenv}}/config/kaf/config.template": "config/kaf/config"
   },
   "shell": {
+    "init_hook": [
+      "envsubst < {{.Virtenv}}/config/kafkactl/config.yaml.template > {{.Virtenv}}/config/kafkactl/config.yaml",
+      "envsubst < {{.Virtenv}}/config/kcat/kcat.conf.template > {{.Virtenv}}/config/kcat/kcat.conf",
+      "envsubst < {{.Virtenv}}/config/kaf/config.template > {{.Virtenv}}/config/kaf/config"
+    ],
     "scripts": {
       "kafka-local-readme": "{{.Virtenv}}/bin/kafka-local-readme"
     }