From 8e75b875e6e7addb24c98a1335c69d128d837a25 Mon Sep 17 00:00:00 2001 From: Sergey Tarasov Date: Fri, 10 May 2024 15:49:32 +0300 Subject: [PATCH 1/4] Redis updates --- docs/redis.md | 32 ++++++++++++++++++++++++-------- templates/memcached.yml | 6 +++--- templates/postgres.yml | 2 +- templates/redis.yml | 6 +++--- templates/redis2.yml | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 templates/redis2.yml diff --git a/docs/redis.md b/docs/redis.md index 9f826a7c..3fad7e20 100644 --- a/docs/redis.md +++ b/docs/redis.md @@ -1,6 +1,22 @@ -# Migrating Redis database from Heroku infrastructure +# Migrating Redis databases -**General considerations:** +There are two templates examples in this repo: +- `redis` - basic non-persistent template. It is good for review-apps or staging or where no persistence is required +- `redis2` - basic persistent template. Good for production where persistence is needed, but cluster is overkill. + +## Option 1: use SLAVEOF (easier way) + +1. create a redis workload that will accept data +2. execute `SLAVEOF source_host source_port`, if needed use `masterauth` to provide auth details +3. wait for replication to pick up all changes (usually very quick), use `INFO` or `DBSIZE` to check progress +4. stop app completely and ensure nothing is writing to any of redises +5. execute `SLAVEOF no one` to disconnect replication +6. switch `REDIS_URL` in the app to point to new server +7. start the app + +## Option 2: use Redis-RIOT (harder way, where option 1 is not possible) + +### General considerations: 1. Heroku uses self-signed TLS certificates, which are not verifiable. It needs special handling by setting TLS verification to `none`, otherwise most apps are not able to connect. @@ -9,7 +25,7 @@ TLS verification to `none`, otherwise most apps are not able to connect. The tool that satisfies those criteria is [Redis-RIOT](https://developer.redis.com/riot/riot-redis/index.html) -**Heroku Redis:** +### Heroku Redis: As Redis-RIOT says, master redis should have keyspace-notifications set to `KA` to be able to do live replication. To do that: @@ -23,7 +39,7 @@ Connect to heroku Redis CLI: heroku redis:cli -a my-app ``` -**Control Plane Redis:** +### Control Plane Redis: Connect to Control Plane Redis CLI: @@ -36,10 +52,10 @@ apt-get update apt-get install redis -y # connect to local cloud Redis -redis-cli -u MY_CONTROL_PLANE_REDIS_URL +redis-cli -u MY_CONTROL_PLANE_REDIS_URL -p 6379 ``` -**Useful Redis CLI commands:** +### Useful Redis CLI commands: Quick-check keys qty: ``` @@ -49,7 +65,7 @@ info keyspace db0:keys=9496,expires=2941,avg_ttl=77670114535 ``` -**Create a Control Plane sync workload** +### Create a Control Plane sync workload ``` name: riot-redis @@ -76,7 +92,7 @@ command args: live ``` -**Sync process** +### Sync process 1. open 1st terminal window with heroku redis CLI, check keys qty 2. open 2nd terminal window with controlplane redis CLI, check keys qty diff --git a/templates/memcached.yml b/templates/memcached.yml index fa3c330f..59f30d48 100644 --- a/templates/memcached.yml +++ b/templates/memcached.yml @@ -4,8 +4,8 @@ spec: type: standard containers: - name: memcached - cpu: 3m - memory: 10Mi + cpu: 25m + memory: 32Mi args: - "-l" - 0.0.0.0 @@ -15,7 +15,7 @@ spec: protocol: tcp defaultOptions: autoscaling: - metric: latency + metric: disabled minScale: 1 maxScale: 1 capacityAI: false diff --git a/templates/postgres.yml b/templates/postgres.yml index 4b609542..d3008862 100644 --- a/templates/postgres.yml +++ b/templates/postgres.yml @@ -23,7 +23,7 @@ spec: uri: "scratch://postgres-vol" defaultOptions: autoscaling: - metric: latency + metric: disabled minScale: 1 maxScale: 1 capacityAI: false diff --git a/templates/redis.yml b/templates/redis.yml index deeb58c2..e54a1d6c 100644 --- a/templates/redis.yml +++ b/templates/redis.yml @@ -4,15 +4,15 @@ spec: type: standard containers: - name: redis - cpu: 3m - memory: 20Mi + cpu: 25m + memory: 32Mi image: "redis:latest" ports: - number: 6379 protocol: tcp defaultOptions: autoscaling: - metric: latency + metric: disabled minScale: 1 maxScale: 1 capacityAI: false diff --git a/templates/redis2.yml b/templates/redis2.yml new file mode 100644 index 00000000..b07ee55f --- /dev/null +++ b/templates/redis2.yml @@ -0,0 +1,37 @@ +kind: volumeset +name: redis-data +spec: + fileSystemType: ext4 + initialCapacity: 10 + performanceClass: general-purpose-ssd +--- +kind: workload +name: redis2 +spec: + type: stateful + containers: + - name: redis + args: + - '--appendonly' + - 'yes' + - '--maxmemory' + - 25mb + cpu: 25m + memory: 32Mi + image: "redis:latest" + ports: + - number: 6379 + protocol: tcp + volumes: + - path: /data + recoveryPolicy: retain + uri: cpln://volumeset/redis-data + defaultOptions: + autoscaling: + metric: disabled + minScale: 1 + maxScale: 1 + capacityAI: false + firewallConfig: + internal: + inboundAllowType: same-gvc From bc6a180ca0fbf1648228140907318ae82918b30f Mon Sep 17 00:00:00 2001 From: Justin Gordon Date: Sat, 11 May 2024 16:20:06 -1000 Subject: [PATCH 2/4] Update docs/redis.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/redis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/redis.md b/docs/redis.md index 3fad7e20..c1bba14b 100644 --- a/docs/redis.md +++ b/docs/redis.md @@ -8,7 +8,7 @@ There are two templates examples in this repo: 1. create a redis workload that will accept data 2. execute `SLAVEOF source_host source_port`, if needed use `masterauth` to provide auth details -3. wait for replication to pick up all changes (usually very quick), use `INFO` or `DBSIZE` to check progress +3. wait for replication to pick up all changes (usually quickly), use `INFO` or `DBSIZE` to check progress 4. stop app completely and ensure nothing is writing to any of redises 5. execute `SLAVEOF no one` to disconnect replication 6. switch `REDIS_URL` in the app to point to new server From 68ea5906122df182c6632c96137141c3ac6b202b Mon Sep 17 00:00:00 2001 From: Justin Gordon Date: Sat, 11 May 2024 16:20:33 -1000 Subject: [PATCH 3/4] Update docs/redis.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/redis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/redis.md b/docs/redis.md index c1bba14b..5f5f8961 100644 --- a/docs/redis.md +++ b/docs/redis.md @@ -19,7 +19,7 @@ There are two templates examples in this repo: ### General considerations: 1. Heroku uses self-signed TLS certificates, which are not verifiable. It needs special handling by setting -TLS verification to `none`, otherwise most apps are not able to connect. +TLS verification to `none`; otherwise, most apps cannot connect. 2. We are moving to private Redis that don't have a public URL, so have to do it from a Control Plane GVC container. From 58dd5605617ecf0fab35012177e8119015a225d3 Mon Sep 17 00:00:00 2001 From: Justin Gordon Date: Sat, 11 May 2024 16:51:00 -1000 Subject: [PATCH 4/4] Update docs/redis.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/redis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/redis.md b/docs/redis.md index 5f5f8961..3ba05b1b 100644 --- a/docs/redis.md +++ b/docs/redis.md @@ -19,7 +19,7 @@ There are two templates examples in this repo: ### General considerations: 1. Heroku uses self-signed TLS certificates, which are not verifiable. It needs special handling by setting -TLS verification to `none`; otherwise, most apps cannot connect. +The tool that satisfies those criteria is [Redis-RIOT](https://developer.redis.com/riot/riot-redis/index.html) 2. We are moving to private Redis that don't have a public URL, so have to do it from a Control Plane GVC container.