From 3a12d2ef3912e11131b8a7f36beef58f405e0e4f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:25:59 +0200 Subject: [PATCH 01/35] Adds schema registry --- 60schema-registry.yml | 22 ++++++++++++++++++++++ 61schema-registry-service.yml | 10 ++++++++++ 2 files changed, 32 insertions(+) create mode 100644 60schema-registry.yml create mode 100644 61schema-registry-service.yml diff --git a/60schema-registry.yml b/60schema-registry.yml new file mode 100644 index 00000000..e02fe8b8 --- /dev/null +++ b/60schema-registry.yml @@ -0,0 +1,22 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: schema-registry + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: schema-registry + spec: + containers: + - name: schema-registry + image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c + env: + - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL + value: zookeeper:2181 + - name: SCHEMA_REGISTRY_HOST_NAME + value: schema-registry + ports: + - containerPort: 8081 diff --git a/61schema-registry-service.yml b/61schema-registry-service.yml new file mode 100644 index 00000000..652b224a --- /dev/null +++ b/61schema-registry-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: schema-registry + namespace: kafka +spec: + ports: + - port: 8081 + selector: + app: schema-registry From c14d4cce6c391443ba0dd4345d06426c510fd233 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:43:33 +0200 Subject: [PATCH 02/35] Avoids the schema-registry service name, becuase it interferes ... with container configuration, as Kubernetes creates the env SCHEMA_REGISTRY_PORT if the service has been created first. --- 61schema-registry-service.yml => 60schemas-service.yml | 2 +- 60schema-registry.yml => 61schemas.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename 61schema-registry-service.yml => 60schemas-service.yml (83%) rename 60schema-registry.yml => 61schemas.yml (85%) diff --git a/61schema-registry-service.yml b/60schemas-service.yml similarity index 83% rename from 61schema-registry-service.yml rename to 60schemas-service.yml index 652b224a..2d1894cd 100644 --- a/61schema-registry-service.yml +++ b/60schemas-service.yml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: schema-registry + name: schemas namespace: kafka spec: ports: diff --git a/60schema-registry.yml b/61schemas.yml similarity index 85% rename from 60schema-registry.yml rename to 61schemas.yml index e02fe8b8..2423e915 100644 --- a/60schema-registry.yml +++ b/61schemas.yml @@ -1,7 +1,7 @@ apiVersion: apps/v1beta1 kind: Deployment metadata: - name: schema-registry + name: schemas namespace: kafka spec: replicas: 1 @@ -11,12 +11,12 @@ spec: app: schema-registry spec: containers: - - name: schema-registry + - name: cp-schema-registry image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c env: - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL value: zookeeper:2181 - name: SCHEMA_REGISTRY_HOST_NAME - value: schema-registry + value: schemas ports: - containerPort: 8081 From 55f5b317df9cdf85d1f8027b00650e51694a372c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:48:16 +0200 Subject: [PATCH 03/35] This is http so there's no reason to not use port 80 --- 60schemas-service.yml | 2 +- 61schemas.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/60schemas-service.yml b/60schemas-service.yml index 2d1894cd..2c30905e 100644 --- a/60schemas-service.yml +++ b/60schemas-service.yml @@ -5,6 +5,6 @@ metadata: namespace: kafka spec: ports: - - port: 8081 + - port: 80 selector: app: schema-registry diff --git a/61schemas.yml b/61schemas.yml index 2423e915..20d18689 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -18,5 +18,7 @@ spec: value: zookeeper:2181 - name: SCHEMA_REGISTRY_HOST_NAME value: schemas + - name: SCHEMA_REGISTRY_LISTENERS + value: http://0.0.0.0:80 ports: - - containerPort: 8081 + - containerPort: 80 From dfc65f1ce19fb93d6a5af4ce52a3cc35acd35e25 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:51:14 +0200 Subject: [PATCH 04/35] Adds kafka-rest proxy as rest:80 --- 70rest-service.yml | 10 ++++++++++ 71rest.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 70rest-service.yml create mode 100644 71rest.yml diff --git a/70rest-service.yml b/70rest-service.yml new file mode 100644 index 00000000..a3ffda6d --- /dev/null +++ b/70rest-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: rest + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: kafka-rest diff --git a/71rest.yml b/71rest.yml new file mode 100644 index 00000000..8f52b429 --- /dev/null +++ b/71rest.yml @@ -0,0 +1,26 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rest + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: kafka-rest + spec: + containers: + - name: cp-kafka-rest + image: confluentinc/cp-kafka-rest@sha256:aa213c1a67eae6ce9836b52a9b5ecee4d6a0b44f2b9cc69f4e4de85131462f1d + env: + - name: KAFKA_REST_ZOOKEEPER_CONNECT + value: zookeeper:2181 + - name: KAFKA_REST_HOST_NAME + value: rest + - name: KAFKA_REST_LISTENERS + value: http://0.0.0.0:80 + - name: KAFKA_REST_SCHEMA_REGISTRY_URL + value: http://schemas.kafka.svc.cluster.local:80 + ports: + - containerPort: 80 From d340d41cc7eda1c44dd069257fa7eabfbc01ec03 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 05:52:14 +0200 Subject: [PATCH 05/35] Tests rest-proxy, but hangs on GET /consumers/.../records --- test/70rest-test1.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 test/70rest-test1.yml diff --git a/test/70rest-test1.yml b/test/70rest-test1.yml new file mode 100644 index 00000000..e52da30a --- /dev/null +++ b/test/70rest-test1.yml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: rest-test1 + namespace: kafka +spec: + template: + metadata: + name: rest-test1 + spec: + containers: + - name: curl + image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874 + env: + - name: REST + value: http://rest.kafka.svc.cluster.local + - name: TOPIC + value: test1 + command: + - /bin/sh + - -c + - > + curl -I $REST; + curl -X POST $REST/consumers/list; + + # http://docs.confluent.io/current/kafka-rest/docs/intro.html + + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + + curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + + curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + + sleep 300 + restartPolicy: Never From 8b1ddad86952bbe70702fb265487c70cbafbf6bd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 06:14:41 +0200 Subject: [PATCH 06/35] Adds more tests, and retries --- test/70rest-test1.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/70rest-test1.yml b/test/70rest-test1.yml index e52da30a..ce7b0190 100644 --- a/test/70rest-test1.yml +++ b/test/70rest-test1.yml @@ -20,10 +20,13 @@ spec: - /bin/sh - -c - > - curl -I $REST; - curl -X POST $REST/consumers/list; + curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; - # http://docs.confluent.io/current/kafka-rest/docs/intro.html + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; From 4fc6880c4ef53690a16a63ac1157d0f073ddbb4c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Jul 2017 15:54:18 +0200 Subject: [PATCH 07/35] Starts a test case for rest-proxy --- test/rest-curl.yml | 117 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 test/rest-curl.yml diff --git a/test/rest-curl.yml b/test/rest-curl.yml new file mode 100644 index 00000000..db5758ac --- /dev/null +++ b/test/rest-curl.yml @@ -0,0 +1,117 @@ +--- +kind: ConfigMap +metadata: + name: rest-curl + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + # Don't complete startup until rest proxy is up and running + curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; + + + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; + + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + + curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + + curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + + tail -f /tmp/testlog + + continue.sh: |- + exit 0 + + run.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rest-curl + namespace: test-kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - test-rest-curl + - --partitions + - "1" + - --replication-factor + - "1" + restartPolicy: Never +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rest-curl + namespace: test-kafka +spec: + replicas: 1 + template: + metadata: + labels: + test-target: kafka + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf + env: + - name: REST + value: http://rest.kafka.svc.cluster.local + - name: TOPIC + value: test-rest-curl + # Test set up + command: + - /bin/bash + - -e + - /test/setup.sh + # Test run, again and again + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/run.sh + # We haven't worked on timing + periodSeconds: 60 + # Test quit on nonzero exit + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/continue.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: rest-curl From 713743ec4469dd40b63344fe7b3d81f6dce5a643 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Jul 2017 16:51:42 +0200 Subject: [PATCH 08/35] We should probably wait for CP 3.3.0 because ... I'm not getting the expected behavior. Might also be because of low resources on minikube, but 3.3.0 should only be a few days away. --- test/rest-curl.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index db5758ac..05fc178d 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -9,17 +9,26 @@ data: setup.sh: |- touch /tmp/testlog - # Don't complete startup until rest proxy is up and running - curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; + # Keep starting up until rest proxy is up and running + curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST - curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics + echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; - curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC + echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ + --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ + $REST/topics/$TOPIC -v --max-time 30 \ + || echo " (timeout might be ok because we only want to send one message)" + # TODO why does the above block? + + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions + echo "" curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; From f5876226d96bc355370f76fbbc8437eae1f213e2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Jul 2017 20:46:25 +0200 Subject: [PATCH 09/35] Sample config from v3.3.0 source --- 11confluent-config.yml | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 11confluent-config.yml diff --git a/11confluent-config.yml b/11confluent-config.yml new file mode 100644 index 00000000..8b4087c8 --- /dev/null +++ b/11confluent-config.yml @@ -0,0 +1,72 @@ +kind: ConfigMap +metadata: + name: confluent-config + namespace: kafka +apiVersion: v1 +data: + schema-registry.properties: |- + # Copyright 2014 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + listeners=http://0.0.0.0:8081 + kafkastore.connection.url=localhost:2181 + kafkastore.topic=_schemas + debug=false + + kafka-rest.properties: |- + ## + # Copyright 2015 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + ## + + #id=kafka-rest-test-server + #schema.registry.url=http://localhost:8081 + #zookeeper.connect=localhost:2181 + # + # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center + # Make sure that monitoring-interceptors-.jar is on the Java class path + #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor + #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor + + log4j.properties: |- + log4j.rootLogger=INFO, stdout, file + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.logger.kafka=ERROR, stdout + log4j.logger.org.apache.zookeeper=ERROR, stdout + log4j.logger.org.apache.kafka=ERROR, stdout + log4j.logger.org.I0Itec.zkclient=ERROR, stdout + log4j.additivity.kafka.server=false + log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false + + log4j.appender.file=org.apache.log4j.RollingFileAppender + log4j.appender.file.maxBackupIndex=10 + log4j.appender.file.maxFileSize=100MB + log4j.appender.file.File=${schema-registry.log.dir}/schema-registry.log + log4j.appender.file.layout=org.apache.log4j.PatternLayout + log4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c)%n From 2d4401bc1fd90cc9af82599517d4a1df7a103ebb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Jul 2017 20:47:33 +0200 Subject: [PATCH 10/35] Remove file appender as docker implies stdout also the sample is from schema-registry.properties while kafka-rest had only the stdout appender (and no per-package levels) --- 11confluent-config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index 8b4087c8..62d89932 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -51,7 +51,7 @@ data: #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor log4j.properties: |- - log4j.rootLogger=INFO, stdout, file + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout @@ -63,10 +63,3 @@ data: log4j.logger.org.I0Itec.zkclient=ERROR, stdout log4j.additivity.kafka.server=false log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false - - log4j.appender.file=org.apache.log4j.RollingFileAppender - log4j.appender.file.maxBackupIndex=10 - log4j.appender.file.maxFileSize=100MB - log4j.appender.file.File=${schema-registry.log.dir}/schema-registry.log - log4j.appender.file.layout=org.apache.log4j.PatternLayout - log4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c)%n From 47910682edfadffccbf77b3f0b2722376dc37350 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 07:35:06 +0200 Subject: [PATCH 11/35] New image uses config files instead of env --- 11confluent-config.yml | 9 +++++---- 61schemas.yml | 19 +++++++++++++------ 71rest.yml | 18 ++++++++++-------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index 62d89932..df916773 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -19,8 +19,8 @@ data: # See the License for the specific language governing permissions and # limitations under the License. - listeners=http://0.0.0.0:8081 - kafkastore.connection.url=localhost:2181 + listeners=http://0.0.0.0:80 + kafkastore.connection.url=zookeeper:2181 kafkastore.topic=_schemas debug=false @@ -42,8 +42,9 @@ data: ## #id=kafka-rest-test-server - #schema.registry.url=http://localhost:8081 - #zookeeper.connect=localhost:2181 + listeners=http://0.0.0.0:80 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + zookeeper.connect=zookeeper:2181 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/61schemas.yml b/61schemas.yml index 20d18689..8d098dd1 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -11,14 +11,21 @@ spec: app: schema-registry spec: containers: - - name: cp-schema-registry - image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c + - name: cp + image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 env: - - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL - value: zookeeper:2181 + # what does this do? - name: SCHEMA_REGISTRY_HOST_NAME value: schemas - - name: SCHEMA_REGISTRY_LISTENERS - value: http://0.0.0.0:80 + command: + - schema-registry-start + - /etc/schema-registry/schema-registry.properties ports: - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/schema-registry + volumes: + - name: config + configMap: + name: confluent-config diff --git a/71rest.yml b/71rest.yml index 8f52b429..987e0131 100644 --- a/71rest.yml +++ b/71rest.yml @@ -11,16 +11,18 @@ spec: app: kafka-rest spec: containers: - - name: cp-kafka-rest - image: confluentinc/cp-kafka-rest@sha256:aa213c1a67eae6ce9836b52a9b5ecee4d6a0b44f2b9cc69f4e4de85131462f1d + - name: cp + image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 env: - - name: KAFKA_REST_ZOOKEEPER_CONNECT - value: zookeeper:2181 + # what does this do? - name: KAFKA_REST_HOST_NAME value: rest - - name: KAFKA_REST_LISTENERS - value: http://0.0.0.0:80 - - name: KAFKA_REST_SCHEMA_REGISTRY_URL - value: http://schemas.kafka.svc.cluster.local:80 ports: - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/kafka-rest + volumes: + - name: config + configMap: + name: confluent-config From 86183d83eb715c97e8647cd16cd963ead542f512 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 10:17:39 +0200 Subject: [PATCH 12/35] Runtime conf is logged at start, which documents host.name in case we need it --- 61schemas.yml | 6 +----- 71rest.yml | 9 ++++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/61schemas.yml b/61schemas.yml index 8d098dd1..4fe7097e 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -12,11 +12,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 - env: - # what does this do? - - name: SCHEMA_REGISTRY_HOST_NAME - value: schemas + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf command: - schema-registry-start - /etc/schema-registry/schema-registry.properties diff --git a/71rest.yml b/71rest.yml index 987e0131..e237357e 100644 --- a/71rest.yml +++ b/71rest.yml @@ -12,11 +12,10 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 - env: - # what does this do? - - name: KAFKA_REST_HOST_NAME - value: rest + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + command: + - kafka-rest-start + - /etc/kafka-rest/kafka-rest.properties ports: - containerPort: 80 volumeMounts: From eb1a1020055ecca572d0dac9a9d5bfb853b3d722 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 12:16:53 +0200 Subject: [PATCH 13/35] Once again be explicit (=unsurprising) about log config path --- 61schemas.yml | 3 +++ 71rest.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/61schemas.yml b/61schemas.yml index 4fe7097e..f85d74c1 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -13,6 +13,9 @@ spec: containers: - name: cp image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: SCHEMA_REGISTRY_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties command: - schema-registry-start - /etc/schema-registry/schema-registry.properties diff --git a/71rest.yml b/71rest.yml index e237357e..f03f5b5b 100644 --- a/71rest.yml +++ b/71rest.yml @@ -13,6 +13,9 @@ spec: containers: - name: cp image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: KAFKAREST_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties command: - kafka-rest-start - /etc/kafka-rest/kafka-rest.properties From f29e6eb8a20edebce65a3ecca792ce28ba602492 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 13:03:55 +0200 Subject: [PATCH 14/35] Logs, unlike docs, revealed that kafka-rest needs ... both zookeeper and bootstrap addresses. Was getting connection errors to localhost:9092 broker id -1, resulting in REST requests never returning. --- 11confluent-config.yml | 3 ++- test/rest-curl.yml | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index df916773..12fa7dbe 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -43,8 +43,9 @@ data: #id=kafka-rest-test-server listeners=http://0.0.0.0:80 - schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 zookeeper.connect=zookeeper:2181 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 05fc178d..7ee719a0 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -22,10 +22,7 @@ data: curl -X POST \ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ - $REST/topics/$TOPIC -v --max-time 30 \ - || echo " (timeout might be ok because we only want to send one message)" - # TODO why does the above block? - + $REST/topics/$TOPIC curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions echo "" From ac5c75a977c796313dc5f84831aa50529457be38 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 14:34:53 +0200 Subject: [PATCH 15/35] Tests commands succeed now. Remains to add asserts. --- test/rest-curl.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 7ee719a0..f8dc8deb 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -12,28 +12,43 @@ data: # Keep starting up until rest proxy is up and running curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/brokers | egrep '."brokers":.0' - curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC echo "" curl -X POST \ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ $REST/topics/$TOPIC + echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_json_consumer + echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription \ + -w "%{http_code}" + echo "" - curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + curl -X GET \ + -H "Accept: application/vnd.kafka.json.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/records - curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + curl -X DELETE \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance tail -f /tmp/testlog From caf56c71bd36be27f1626e1849889a6b893035d4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 14:54:25 +0200 Subject: [PATCH 16/35] Looks like Schema Registry works with empty bootstrap servers --- 11confluent-config.yml | 1 + test/rest-curl.yml | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/11confluent-config.yml b/11confluent-config.yml index 12fa7dbe..b56484b7 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -20,6 +20,7 @@ data: # limitations under the License. listeners=http://0.0.0.0:80 + #kafkastore.bootstrap.servers kafkastore.connection.url=zookeeper:2181 kafkastore.topic=_schemas debug=false diff --git a/test/rest-curl.yml b/test/rest-curl.yml index f8dc8deb..5c6489df 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -50,6 +50,27 @@ data: -H "Content-Type: application/vnd.kafka.v2+json" \ $REST/consumers/my_json_consumer/instances/my_consumer_instance + # schema-registry + + curl -X GET $SCHEMAS/subjects + echo "" + + curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \ + --data '{"schema": "{\"type\": \"string\"}"}' \ + $SCHEMAS/subjects/$TOPIC-key/versions + echo "" + + curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \ + --data '{"schema": "{\"type\": \"string\"}"}' \ + $SCHEMAS/subjects/$TOPIC-value/versions + echo "" + + curl -X GET $SCHEMAS/schemas/ids/1 + echo "" + + curl -X GET $SCHEMAS/subjects/$TOPIC-value/versions/1 + echo "" + tail -f /tmp/testlog continue.sh: |- @@ -104,6 +125,8 @@ spec: - name: testcase image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf env: + - name: SCHEMAS + value: http://schemas.kafka.svc.cluster.local - name: REST value: http://rest.kafka.svc.cluster.local - name: TOPIC From ff698326390c191a39c95b38d019f7b306fb0063 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 15:05:06 +0200 Subject: [PATCH 17/35] Demonstrates that rest+schemas work together --- test/rest-curl.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 5c6489df..998a36f0 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -71,6 +71,27 @@ data: curl -X GET $SCHEMAS/subjects/$TOPIC-value/versions/1 echo "" + # rest + schema + # TODO new topic needed because this breaks json consumer above + + curl -X POST -H "Content-Type: application/vnd.kafka.avro.v2+json" \ + -H "Accept: application/vnd.kafka.v2+json" \ + --data '{"value_schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}", "records": [{"value": {"name": "testUser"}}]}' \ + $REST/topics/$TOPIC + echo "" + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "avro", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_avro_consumer + echo "" + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_avro_consumer/instances/my_consumer_instance/subscription + + curl -X GET -H "Accept: application/vnd.kafka.avro.v2+json" \ + $REST/consumers/my_avro_consumer/instances/my_consumer_instance/records + tail -f /tmp/testlog continue.sh: |- From edeca3df36e72ddb464a10c580ae33b8ed1d2f17 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:22:42 +0200 Subject: [PATCH 18/35] Moves to an addon folder, kubectl apply -f addon-cp/ --- 11confluent-config.yml => addon-cp/confluent-config.yml | 0 70rest-service.yml => addon-cp/rest-service.yml | 0 71rest.yml => addon-cp/rest.yml | 0 60schemas-service.yml => addon-cp/schemas-service.yml | 0 61schemas.yml => addon-cp/schemas.yml | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename 11confluent-config.yml => addon-cp/confluent-config.yml (100%) rename 70rest-service.yml => addon-cp/rest-service.yml (100%) rename 71rest.yml => addon-cp/rest.yml (100%) rename 60schemas-service.yml => addon-cp/schemas-service.yml (100%) rename 61schemas.yml => addon-cp/schemas.yml (100%) diff --git a/11confluent-config.yml b/addon-cp/confluent-config.yml similarity index 100% rename from 11confluent-config.yml rename to addon-cp/confluent-config.yml diff --git a/70rest-service.yml b/addon-cp/rest-service.yml similarity index 100% rename from 70rest-service.yml rename to addon-cp/rest-service.yml diff --git a/71rest.yml b/addon-cp/rest.yml similarity index 100% rename from 71rest.yml rename to addon-cp/rest.yml diff --git a/60schemas-service.yml b/addon-cp/schemas-service.yml similarity index 100% rename from 60schemas-service.yml rename to addon-cp/schemas-service.yml diff --git a/61schemas.yml b/addon-cp/schemas.yml similarity index 100% rename from 61schemas.yml rename to addon-cp/schemas.yml From 4ee79a4e25325aad52ced1c771fe4e45316d0af8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:24:32 +0200 Subject: [PATCH 19/35] Stay online at update, even with only one replica --- addon-cp/rest.yml | 5 +++++ addon-cp/schemas.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index f03f5b5b..a450eab7 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -5,6 +5,11 @@ metadata: namespace: kafka spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 template: metadata: labels: diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index f85d74c1..00bd62f5 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -5,6 +5,11 @@ metadata: namespace: kafka spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 template: metadata: labels: From 2f946b26f308d2ee6e3eb2b0d307c30b98a462a4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:35:09 +0200 Subject: [PATCH 20/35] Adds readiness probes --- addon-cp/rest.yml | 4 ++++ addon-cp/schemas.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index a450eab7..a08587d8 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -24,6 +24,10 @@ spec: command: - kafka-rest-start - /etc/kafka-rest/kafka-rest.properties + readinessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index 00bd62f5..82656408 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -24,6 +24,10 @@ spec: command: - schema-registry-start - /etc/schema-registry/schema-registry.properties + readinessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: From c20adc0a6cfc54e91db66271ef7b7481d4ff7f0a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:36:04 +0200 Subject: [PATCH 21/35] Adds liveness probes identical to readiness --- addon-cp/rest.yml | 4 ++++ addon-cp/schemas.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index a08587d8..60fcc31b 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -28,6 +28,10 @@ spec: httpGet: path: / port: 80 + livenessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index 82656408..4d99ce07 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -28,6 +28,10 @@ spec: httpGet: path: / port: 80 + livenessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: From 3b6a11ecfd707e738b6e65c4c179048e77336dfd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 3 Oct 2017 15:29:59 +0200 Subject: [PATCH 22/35] Confluent platform built from same java image as kafka:0.11.0.1 --- addon-cp/rest.yml | 2 +- addon-cp/schemas.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index 60fcc31b..c835ebf4 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1 env: - name: KAFKAREST_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index 4d99ce07..c762d63a 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1 env: - name: SCHEMA_REGISTRY_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties From 3571677465894288670e1e9668c8ed1d3bf824f8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 29 Nov 2017 06:54:29 +0100 Subject: [PATCH 23/35] Adapts existing addon to 3.0+ folder structure --- {addon-cp => confluent-rest}/confluent-config.yml | 0 {addon-cp => confluent-rest}/rest-service.yml | 0 {addon-cp => confluent-rest}/rest.yml | 0 {addon-cp => confluent-rest}/schemas-service.yml | 0 {addon-cp => confluent-rest}/schemas.yml | 0 {test => confluent-rest/test}/70rest-test1.yml | 0 {test => confluent-rest/test}/rest-curl.yml | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename {addon-cp => confluent-rest}/confluent-config.yml (100%) rename {addon-cp => confluent-rest}/rest-service.yml (100%) rename {addon-cp => confluent-rest}/rest.yml (100%) rename {addon-cp => confluent-rest}/schemas-service.yml (100%) rename {addon-cp => confluent-rest}/schemas.yml (100%) rename {test => confluent-rest/test}/70rest-test1.yml (100%) rename {test => confluent-rest/test}/rest-curl.yml (100%) diff --git a/addon-cp/confluent-config.yml b/confluent-rest/confluent-config.yml similarity index 100% rename from addon-cp/confluent-config.yml rename to confluent-rest/confluent-config.yml diff --git a/addon-cp/rest-service.yml b/confluent-rest/rest-service.yml similarity index 100% rename from addon-cp/rest-service.yml rename to confluent-rest/rest-service.yml diff --git a/addon-cp/rest.yml b/confluent-rest/rest.yml similarity index 100% rename from addon-cp/rest.yml rename to confluent-rest/rest.yml diff --git a/addon-cp/schemas-service.yml b/confluent-rest/schemas-service.yml similarity index 100% rename from addon-cp/schemas-service.yml rename to confluent-rest/schemas-service.yml diff --git a/addon-cp/schemas.yml b/confluent-rest/schemas.yml similarity index 100% rename from addon-cp/schemas.yml rename to confluent-rest/schemas.yml diff --git a/test/70rest-test1.yml b/confluent-rest/test/70rest-test1.yml similarity index 100% rename from test/70rest-test1.yml rename to confluent-rest/test/70rest-test1.yml diff --git a/test/rest-curl.yml b/confluent-rest/test/rest-curl.yml similarity index 100% rename from test/rest-curl.yml rename to confluent-rest/test/rest-curl.yml From ed29106874420ba1faafccd12632a461ff40ecc8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 29 Nov 2017 07:27:46 +0100 Subject: [PATCH 24/35] Upgrades to 4.0.0 --- confluent-rest/confluent-config.yml | 8 +++----- confluent-rest/rest-service.yml | 2 +- confluent-rest/rest.yml | 4 ++-- confluent-rest/schemas.yml | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/confluent-rest/confluent-config.yml b/confluent-rest/confluent-config.yml index b56484b7..dd77c952 100644 --- a/confluent-rest/confluent-config.yml +++ b/confluent-rest/confluent-config.yml @@ -20,8 +20,7 @@ data: # limitations under the License. listeners=http://0.0.0.0:80 - #kafkastore.bootstrap.servers - kafkastore.connection.url=zookeeper:2181 + kafkastore.bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 kafkastore.topic=_schemas debug=false @@ -44,9 +43,8 @@ data: #id=kafka-rest-test-server listeners=http://0.0.0.0:80 - bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - zookeeper.connect=zookeeper:2181 - schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 + schema.registry.url=http://schemas.kafka:80 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/confluent-rest/rest-service.yml b/confluent-rest/rest-service.yml index a3ffda6d..524f401c 100644 --- a/confluent-rest/rest-service.yml +++ b/confluent-rest/rest-service.yml @@ -7,4 +7,4 @@ spec: ports: - port: 80 selector: - app: kafka-rest + app: rest-proxy diff --git a/confluent-rest/rest.yml b/confluent-rest/rest.yml index c835ebf4..71e6c535 100644 --- a/confluent-rest/rest.yml +++ b/confluent-rest/rest.yml @@ -13,11 +13,11 @@ spec: template: metadata: labels: - app: kafka-rest + app: rest-proxy spec: containers: - name: cp - image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1 + image: solsson/kafka-cp@sha256:2797da107f477ede2e826c29b2589f99f22d9efa2ba6916b63e07c7045e15044 env: - name: KAFKAREST_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties diff --git a/confluent-rest/schemas.yml b/confluent-rest/schemas.yml index c762d63a..50e6207f 100644 --- a/confluent-rest/schemas.yml +++ b/confluent-rest/schemas.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1 + image: solsson/kafka-cp@sha256:2797da107f477ede2e826c29b2589f99f22d9efa2ba6916b63e07c7045e15044 env: - name: SCHEMA_REGISTRY_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties From 1833c980474c4aba99b092f8cf08e2829d96e3da Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 29 Nov 2017 07:28:18 +0100 Subject: [PATCH 25/35] Manifests updated for Kubernetes 1.8 --- confluent-rest/rest.yml | 5 ++++- confluent-rest/schemas.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/confluent-rest/rest.yml b/confluent-rest/rest.yml index 71e6c535..cd8e203c 100644 --- a/confluent-rest/rest.yml +++ b/confluent-rest/rest.yml @@ -1,10 +1,13 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: Deployment metadata: name: rest namespace: kafka spec: replicas: 1 + selector: + matchLabels: + app: rest-proxy strategy: type: RollingUpdate rollingUpdate: diff --git a/confluent-rest/schemas.yml b/confluent-rest/schemas.yml index 50e6207f..a78723fe 100644 --- a/confluent-rest/schemas.yml +++ b/confluent-rest/schemas.yml @@ -1,10 +1,13 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: Deployment metadata: name: schemas namespace: kafka spec: replicas: 1 + selector: + matchLabels: + app: schema-registry strategy: type: RollingUpdate rollingUpdate: From 465a5052b629cfca67c6e82926e5f51c23f307d4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 1 Dec 2017 07:51:03 +0100 Subject: [PATCH 26/35] Updates test manifests for 3.0+ --- confluent-rest/test/70rest-test1.yml | 7 ++++--- confluent-rest/test/rest-curl.yml | 12 ++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/confluent-rest/test/70rest-test1.yml b/confluent-rest/test/70rest-test1.yml index ce7b0190..f26cdc96 100644 --- a/confluent-rest/test/70rest-test1.yml +++ b/confluent-rest/test/70rest-test1.yml @@ -4,21 +4,22 @@ metadata: name: rest-test1 namespace: kafka spec: + backoffLimit: 1 template: metadata: name: rest-test1 spec: containers: - name: curl - image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874 + image: solsson/curl@sha256:523319afd39573746e8f5a7c98d4a6cd4b8cbec18b41eb30c8baa13ede120ce3 env: - name: REST value: http://rest.kafka.svc.cluster.local - name: TOPIC value: test1 command: - - /bin/sh - - -c + - /bin/bash + - -ce - > curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; diff --git a/confluent-rest/test/rest-curl.yml b/confluent-rest/test/rest-curl.yml index 998a36f0..486392cf 100644 --- a/confluent-rest/test/rest-curl.yml +++ b/confluent-rest/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: - ./bin/kafka-topics.sh - --zookeeper @@ -129,22 +129,26 @@ spec: - "1" restartPolicy: Never --- -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: Deployment metadata: name: rest-curl namespace: test-kafka spec: replicas: 1 + selector: + matchLabels: + test-target: kafka-confluent-rest + test-type: readiness template: metadata: labels: - test-target: kafka + test-target: kafka-confluent-rest test-type: readiness spec: containers: - name: testcase - image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf + image: solsson/curl@sha256:523319afd39573746e8f5a7c98d4a6cd4b8cbec18b41eb30c8baa13ede120ce3 env: - name: SCHEMAS value: http://schemas.kafka.svc.cluster.local From 4d5815ce9d6eb55770d92931d582eeb57f6776dd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 5 Dec 2017 07:54:24 +0100 Subject: [PATCH 27/35] Adds CORS headers for REST Proxy, and clean up --- confluent-rest/confluent-config.yml | 39 +++-------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/confluent-rest/confluent-config.yml b/confluent-rest/confluent-config.yml index dd77c952..96e211cf 100644 --- a/confluent-rest/confluent-config.yml +++ b/confluent-rest/confluent-config.yml @@ -5,51 +5,20 @@ metadata: apiVersion: v1 data: schema-registry.properties: |- - # Copyright 2014 Confluent Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - listeners=http://0.0.0.0:80 kafkastore.bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 kafkastore.topic=_schemas debug=false kafka-rest.properties: |- - ## - # Copyright 2015 Confluent Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ## - #id=kafka-rest-test-server listeners=http://0.0.0.0:80 bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 schema.registry.url=http://schemas.kafka:80 - # - # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center - # Make sure that monitoring-interceptors-.jar is on the Java class path - #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor - #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor + + # https://github.com/Landoop/kafka-topics-ui#common-issues + access.control.allow.methods=GET,POST,PUT,DELETE,OPTIONS + access.control.allow.origin=* log4j.properties: |- log4j.rootLogger=INFO, stdout From fe57b06cde1b411776af4085eff7855d9674d299 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 5 Dec 2017 08:11:17 +0100 Subject: [PATCH 28/35] Maybe the CORS headers aren't necessary with PROXY=true ... env added to Landoop's services. However, I don't want to add these services in production because I don't understand the license, hence no manifests for them. --- confluent-rest/confluent-config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent-rest/confluent-config.yml b/confluent-rest/confluent-config.yml index 96e211cf..1b4acbf1 100644 --- a/confluent-rest/confluent-config.yml +++ b/confluent-rest/confluent-config.yml @@ -10,6 +10,10 @@ data: kafkastore.topic=_schemas debug=false + # https://github.com/Landoop/schema-registry-ui#prerequisites + access.control.allow.methods=GET,POST,PUT,OPTIONS + access.control.allow.origin=* + kafka-rest.properties: |- #id=kafka-rest-test-server listeners=http://0.0.0.0:80 From 6f4a0be6d11287883c190b9f8f2283764d9b9367 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 17 Dec 2017 21:52:58 +0100 Subject: [PATCH 29/35] Schema Registry and REST Proxy are generic names, but Avro centric --- .../confluent-config.yml => avro-tools/avro-tools-config.yml | 2 +- {confluent-rest => avro-tools}/rest-service.yml | 0 {confluent-rest => avro-tools}/rest.yml | 2 +- {confluent-rest => avro-tools}/schemas-service.yml | 0 {confluent-rest => avro-tools}/schemas.yml | 2 +- {confluent-rest => avro-tools}/test/70rest-test1.yml | 0 {confluent-rest => avro-tools}/test/rest-curl.yml | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename confluent-rest/confluent-config.yml => avro-tools/avro-tools-config.yml (98%) rename {confluent-rest => avro-tools}/rest-service.yml (100%) rename {confluent-rest => avro-tools}/rest.yml (96%) rename {confluent-rest => avro-tools}/schemas-service.yml (100%) rename {confluent-rest => avro-tools}/schemas.yml (96%) rename {confluent-rest => avro-tools}/test/70rest-test1.yml (100%) rename {confluent-rest => avro-tools}/test/rest-curl.yml (100%) diff --git a/confluent-rest/confluent-config.yml b/avro-tools/avro-tools-config.yml similarity index 98% rename from confluent-rest/confluent-config.yml rename to avro-tools/avro-tools-config.yml index 1b4acbf1..8323a0b4 100644 --- a/confluent-rest/confluent-config.yml +++ b/avro-tools/avro-tools-config.yml @@ -1,6 +1,6 @@ kind: ConfigMap metadata: - name: confluent-config + name: avro-tools-config namespace: kafka apiVersion: v1 data: diff --git a/confluent-rest/rest-service.yml b/avro-tools/rest-service.yml similarity index 100% rename from confluent-rest/rest-service.yml rename to avro-tools/rest-service.yml diff --git a/confluent-rest/rest.yml b/avro-tools/rest.yml similarity index 96% rename from confluent-rest/rest.yml rename to avro-tools/rest.yml index cd8e203c..c2aaaa25 100644 --- a/confluent-rest/rest.yml +++ b/avro-tools/rest.yml @@ -43,4 +43,4 @@ spec: volumes: - name: config configMap: - name: confluent-config + name: avro-tools-config diff --git a/confluent-rest/schemas-service.yml b/avro-tools/schemas-service.yml similarity index 100% rename from confluent-rest/schemas-service.yml rename to avro-tools/schemas-service.yml diff --git a/confluent-rest/schemas.yml b/avro-tools/schemas.yml similarity index 96% rename from confluent-rest/schemas.yml rename to avro-tools/schemas.yml index a78723fe..f425e410 100644 --- a/confluent-rest/schemas.yml +++ b/avro-tools/schemas.yml @@ -43,4 +43,4 @@ spec: volumes: - name: config configMap: - name: confluent-config + name: avro-tools-config diff --git a/confluent-rest/test/70rest-test1.yml b/avro-tools/test/70rest-test1.yml similarity index 100% rename from confluent-rest/test/70rest-test1.yml rename to avro-tools/test/70rest-test1.yml diff --git a/confluent-rest/test/rest-curl.yml b/avro-tools/test/rest-curl.yml similarity index 100% rename from confluent-rest/test/rest-curl.yml rename to avro-tools/test/rest-curl.yml From 9686f2e24f6419011c9b486e39a5af0f8f3b85c9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 17 Dec 2017 21:57:58 +0100 Subject: [PATCH 30/35] Renames services too; other registries and rest services may come --- avro-tools/rest-service.yml | 2 +- avro-tools/rest.yml | 2 +- avro-tools/schemas-service.yml | 2 +- avro-tools/schemas.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/avro-tools/rest-service.yml b/avro-tools/rest-service.yml index 524f401c..bf683a32 100644 --- a/avro-tools/rest-service.yml +++ b/avro-tools/rest-service.yml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: rest + name: avro-rest namespace: kafka spec: ports: diff --git a/avro-tools/rest.yml b/avro-tools/rest.yml index c2aaaa25..be299e6a 100644 --- a/avro-tools/rest.yml +++ b/avro-tools/rest.yml @@ -1,7 +1,7 @@ apiVersion: apps/v1beta2 kind: Deployment metadata: - name: rest + name: avro-rest namespace: kafka spec: replicas: 1 diff --git a/avro-tools/schemas-service.yml b/avro-tools/schemas-service.yml index 2c30905e..9a9cd4e9 100644 --- a/avro-tools/schemas-service.yml +++ b/avro-tools/schemas-service.yml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: schemas + name: avro-schemas namespace: kafka spec: ports: diff --git a/avro-tools/schemas.yml b/avro-tools/schemas.yml index f425e410..bd94d6a8 100644 --- a/avro-tools/schemas.yml +++ b/avro-tools/schemas.yml @@ -1,7 +1,7 @@ apiVersion: apps/v1beta2 kind: Deployment metadata: - name: schemas + name: avro-schemas namespace: kafka spec: replicas: 1 From 8def797856867cafd347496baf266cddac444c7c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 17 Dec 2017 22:21:30 +0100 Subject: [PATCH 31/35] Fixes the rename --- avro-tools/avro-tools-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avro-tools/avro-tools-config.yml b/avro-tools/avro-tools-config.yml index 8323a0b4..43fe2682 100644 --- a/avro-tools/avro-tools-config.yml +++ b/avro-tools/avro-tools-config.yml @@ -18,7 +18,7 @@ data: #id=kafka-rest-test-server listeners=http://0.0.0.0:80 bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 - schema.registry.url=http://schemas.kafka:80 + schema.registry.url=http://avro-schemas.kafka:80 # https://github.com/Landoop/kafka-topics-ui#common-issues access.control.allow.methods=GET,POST,PUT,DELETE,OPTIONS From ec5eb7f9bae2b450147623cfbba7b30f97b2750c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 17 Dec 2017 22:49:15 +0100 Subject: [PATCH 32/35] Maybe this port number isn't used, but it shouldn't differ from listeners port --- avro-tools/avro-tools-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/avro-tools/avro-tools-config.yml b/avro-tools/avro-tools-config.yml index 43fe2682..e0d1ace9 100644 --- a/avro-tools/avro-tools-config.yml +++ b/avro-tools/avro-tools-config.yml @@ -5,6 +5,7 @@ metadata: apiVersion: v1 data: schema-registry.properties: |- + port=80 listeners=http://0.0.0.0:80 kafkastore.bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 kafkastore.topic=_schemas From 2499e6b1b3d7161d549beaefa8b033fa83f74f54 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 17 Dec 2017 22:49:31 +0100 Subject: [PATCH 33/35] Displays broker connectivity messages in logs --- avro-tools/avro-tools-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/avro-tools/avro-tools-config.yml b/avro-tools/avro-tools-config.yml index e0d1ace9..f0634ec7 100644 --- a/avro-tools/avro-tools-config.yml +++ b/avro-tools/avro-tools-config.yml @@ -38,3 +38,5 @@ data: log4j.logger.org.I0Itec.zkclient=ERROR, stdout log4j.additivity.kafka.server=false log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false + + log4j.logger.org.apache.kafka.clients.Metadata=DEBUG, stdout From e549959039d69bdc78fbea1be12f6be93bc4cbbd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 17 Dec 2017 23:01:32 +0100 Subject: [PATCH 34/35] Shows "Marking the coordinator x dead" --- avro-tools/avro-tools-config.yml | 1 + avro-tools/schemas.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/avro-tools/avro-tools-config.yml b/avro-tools/avro-tools-config.yml index f0634ec7..9adf3cf0 100644 --- a/avro-tools/avro-tools-config.yml +++ b/avro-tools/avro-tools-config.yml @@ -40,3 +40,4 @@ data: log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false log4j.logger.org.apache.kafka.clients.Metadata=DEBUG, stdout + log4j.logger.org.apache.kafka.clients.consumer.internals.AbstractCoordinator=INFO, stdout diff --git a/avro-tools/schemas.yml b/avro-tools/schemas.yml index bd94d6a8..b811d17d 100644 --- a/avro-tools/schemas.yml +++ b/avro-tools/schemas.yml @@ -35,6 +35,7 @@ spec: httpGet: path: / port: 80 + initialDelaySeconds: 60 ports: - containerPort: 80 volumeMounts: From 4d26f51626ef5c9bd42db9ff8cc7ed0f42e90b2c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 18 Dec 2017 08:23:00 +0100 Subject: [PATCH 35/35] Sample log config hides warnings, we don't --- avro-tools/avro-tools-config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/avro-tools/avro-tools-config.yml b/avro-tools/avro-tools-config.yml index 9adf3cf0..5f5d3425 100644 --- a/avro-tools/avro-tools-config.yml +++ b/avro-tools/avro-tools-config.yml @@ -32,10 +32,10 @@ data: log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - log4j.logger.kafka=ERROR, stdout - log4j.logger.org.apache.zookeeper=ERROR, stdout - log4j.logger.org.apache.kafka=ERROR, stdout - log4j.logger.org.I0Itec.zkclient=ERROR, stdout + log4j.logger.kafka=WARN, stdout + log4j.logger.org.apache.zookeeper=WARN, stdout + log4j.logger.org.apache.kafka=WARN, stdout + log4j.logger.org.I0Itec.zkclient=WARN, stdout log4j.additivity.kafka.server=false log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false