Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cassandra support #17

Merged
merged 40 commits into from
Feb 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
eac0ab6
fix vgo build (#2)
Aug 15, 2018
2a4063e
Init cassandra support on set/get/delete
dispensable Jun 21, 2023
200ec2f
Fix bugs and implement ?key syntax
dispensable Jun 27, 2023
cb54dcd
Add auth support
dispensable Jun 29, 2023
1459d73
Log key when dual write err at one end
dispensable Jun 29, 2023
90859d3
Add more metrics
dispensable Jul 6, 2023
b6af434
Incr connnections
dispensable Jul 7, 2023
c7795cb
Make cstar can used by other project
dispensable Jul 18, 2023
e2c9218
Cstar can set wth bdbvalue
dispensable Jul 18, 2023
3ac0166
Add prefix read support
dispensable Aug 1, 2023
6501c94
Disable gobeansdb when bdb w/r both disabled
dispensable Aug 1, 2023
0856d7e
Remove unused bin file
dispensable Aug 1, 2023
115780c
Add password_file of cassandra cfg support
dispensable Aug 3, 2023
1649b9e
Support switch storage engine by prefix
dispensable Aug 17, 2023
c39abd9
Add tests and use generic trie
dispensable Aug 18, 2023
2dee14a
Only return dual write err when bdb r
dispensable Aug 21, 2023
e2ba00a
Add cfg check for prefix dup
dispensable Aug 21, 2023
b4f34bc
Support reload default storage
dispensable Aug 21, 2023
6558764
Fix style
dispensable Aug 31, 2023
68c8d81
Bump version
dispensable Aug 31, 2023
cf0954a
Tolerant empty prefix/table switch cfg
dispensable Aug 31, 2023
421b738
Default enable beansdb backend
dispensable Sep 4, 2023
915c85d
Err when mem alloc failed
dispensable Sep 6, 2023
eefbccf
Try to fix mem leak
dispensable Sep 6, 2023
5d7ad43
Bump version
dispensable Sep 6, 2023
01fdfa3
Fix memory leak
dispensable Sep 9, 2023
cf0c6e2
Add c* stored dispatcher cfg
dispensable Sep 22, 2023
9b86e8e
Fix successtargets overfide error
dispensable Sep 22, 2023
5d105c1
Support disable dstore/cstar write completely
dispensable Sep 22, 2023
eaede16
Fix empty config crash
dispensable Oct 16, 2023
d7b668a
Improve prometheus histogram accuracy
dispensable Oct 23, 2023
12f30b9
Make get c* cfg prettier
dispensable Oct 25, 2023
ab37a53
Set json type when resp
dispensable Oct 25, 2023
bd23a05
Add dev env and try fix github test
dispensable Jan 8, 2024
4f2d9fc
Fix tests
dispensable Jan 9, 2024
f5be190
Make ci happy
dispensable Jan 9, 2024
cddb750
Fix missing files
dispensable Jan 9, 2024
7c558b3
Add a read only round robin scheduler
dispensable Jan 23, 2024
c101cf9
Add readonly rr scheduler test
dispensable Jan 23, 2024
e3b905f
Verify rwswitch status before update cfg
dispensable Jan 23, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .doubanpde/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
SHELL := /bin/bash
PROJECT_DIR := /home/project

env:
dpi -y -D "-y"
mkdir -p /tmp/gobeansproxy_prefix/proxy/

build:
go build -o gobeansproxy_bin

start-proxy: build
./gobeansproxy_bin -confdir .doubanpde/scripts/bdb/gobeansproxy/prefix-switch-cfg/conf/

start-riven-proxy: build
./gobeansproxy_bin -confdir .doubanpde/scripts/bdb/rivenbeansproxy/conf/

start-proxy-gc-trace: build
GODEBUG=gctrace=1 ./gobeansproxy_bin -confdir .doubanpde/scripts/bdb/gobeansproxy/prefix-switch-cfg/conf/

start-proxy-valgrind: build
G_SLICE=always-malloc G_DEBUG=gc-friendly valgrind -v --tool=memcheck --leak-check=full --num-callers=40 --error-limit=no --log-file=valgrind.log ./gobeansproxy_bin -confdir .doubanpde/scripts/bdb/gobeansproxy/prefix-switch-cfg/conf/

tail-log:
tail -f /tmp/gobeansproxy_prefix/proxy/*.log

cqlsh:
cqlsh -u cassandra -p cassandra
126 changes: 126 additions & 0 deletions .doubanpde/pde.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: "gobeansproxy"
createId: "{{ uuid }}"
createdBy: pdectl
createdByUser: wangqiang
runByUser: '{{ .CliArgs.String "username" }}'
runByPdectlVersion: "{{ .CliArgs.App.Version }}"
runnerAddress: "{{ .RunnerAddress }}"
createdTime: "{{ .CreatedTime }}"
pdeVersion: "v0.1.4"
useWebEditor: "false"
webEditorPort: 0
webEditorType: ""
name: "gobeansproxy"
annotations:
pdectl.douban.com/cfg/exec-cmd: '{{ .CliArgs.String "exec-default-cmd" }}'
spec:
containers:
- name: "main"
env:
- name: HOSTNAME
value: "gobeansproxy-main"
- name: SCRIBE_HOST
value: 10.0.2.2
image: "docker.douban/sa/pde-go-cli:latest-1.20-v2"
ports:
volumeMounts:
# mount go path src to container go path
- mountPath: /go/src/
name: go-path-src
# mount code folder
- mountPath: /home/project/
name: code
- mountPath: /root/
name: userhome
- mountPath: '/home/{{ .CliArgs.String "username" }}'
name: userhome
- mountPath: /fuse:rslave
name: fuse
- mountPath: /etc/douban/
name: etc-douban
readOnly: true
- mountPath: /etc/localtime
name: etc-localtime
readOnly: true
- mountPath: /var/run/nscd/
name: var-run-nscd
readOnly: true
workingDir: /home/project
# - name: mc
# image: docker.douban/memcached:latest
# workingDir: /
{{- range (mkSlice 57980 57981 57982 57983) }}
- name: beansdb-{{ . }}
image: docker.douban/platform/gobeansdb:latest
workingDir: /data/
volumeMounts:
- mountPath: /data
name: beansdb-{{ . }}-data-dir
- mountPath: /gobeansdb/default_beansdb_cfg/
name: beansdb-{{ . }}-cfg-dir
{{- end }}
- name: cassandra
image: docker.douban/dba/cassandra:4.1.2
workingDir: /
volumeMounts:
- mountPath: /var/lib/cassandra/
name: cassandra-data-dir
# - mountPath: /tmp/cassandra/
# name: cassandra-cfg
# command:
# - "/bin/bash"
# args:
# - "-c"
# - >
# cp -rfv /tmp/cassandra/cassandra.yaml /etc/cassandra/ &&
# /usr/local/bin/docker-entrypoint.sh cassandra -f
restartPolicy: Never
volumes:
- hostPath:
path: '{{ expandEnvVar "$GOPATH/src" }}'
type: Directory
name: go-path-src
{{- $env := . }}
{{- range (mkSlice 57980 57981 57982 57983) }}
- hostPath:
path: '{{ $env.CliArgs.String "project-dir" }}/.doubanpde/data/beansdb-{{ . }}/'
type: DirectoryOrCreate
name: beansdb-{{ . }}-data-dir
- hostPath:
path: '{{ $env.CliArgs.String "project-dir" }}/.doubanpde/scripts/bdb/gobeansproxy/{{ . }}/conf/'
type: Directory
name: beansdb-{{ . }}-cfg-dir
{{- end }}
- hostPath:
path: '{{ .CliArgs.String "project-dir" }}/.doubanpde/data/cassandra/'
type: DirectoryOrCreate
name: cassandra-data-dir
- hostPath:
path: '{{ .CliArgs.String "project-dir" }}/.doubanpde/scripts/cassandra/'
name: cassandra-cfg
- hostPath:
path: '{{ .CliArgs.String "project-dir" }}'
type: Directory
name: code
- hostPath:
path: '{{ expandEnvVar "$HOME/" }}'
type: Directory
name: userhome
- hostPath:
path: /fuse
type: Directory
name: fuse
- hostPath:
path: /etc/douban/
name: etc-douban
- hostPath:
path: /etc/localtime
name: etc-localtime
- hostPath:
path: /var/run/nscd/
name: var-run-nscd

32 changes: 32 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57980/conf/global.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
hstore:
data:
check_vhash: true
datafile_max_str: 4000M
flush_interval: 60
flush_wake_str: 10M
no_gc_days: 7
hint:
hint_index_interval_str: 32K
hint_merge_interval: 5
hint_no_merged: true
hint_split_cap_str: 1M
htree:
tree_height: 3
local:
home: /data
mc:
body_big_str: 5M
body_c_str: 0K
body_max_str: 50M
flush_max_str: 100M
max_key_len: 250
max_req: 16
server:
accesslog: /tmp/access.log
errorlog: /tmp/error.log
hostname: 127.0.0.1
listen: 0.0.0.0
port: 57980
threads: 4
webport: 57990
zk: 'NO'
26 changes: 26 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57980/conf/route.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
backup:
- 127.0.0.1:57983
main:
- addr: 127.0.0.1:57980
buckets: &id001
- '0'
- '1'
- '2'
- '3'
- '4'
- '5'
- '6'
- '7'
- '8'
- '9'
- a
- b
- c
- d
- e
- f
- addr: 127.0.0.1:57981
buckets: *id001
- addr: 127.0.0.1:57982
buckets: *id001
numbucket: 16
32 changes: 32 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57981/conf/global.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
hstore:
data:
check_vhash: true
datafile_max_str: 4000M
flush_interval: 60
flush_wake_str: 10M
no_gc_days: 7
hint:
hint_index_interval_str: 32K
hint_merge_interval: 5
hint_no_merged: true
hint_split_cap_str: 1M
htree:
tree_height: 3
local:
home: /data
mc:
body_big_str: 5M
body_c_str: 0K
body_max_str: 50M
flush_max_str: 100M
max_key_len: 250
max_req: 16
server:
accesslog: /tmp/access.log
errorlog: /tmp/error.log
hostname: 127.0.0.1
listen: 0.0.0.0
port: 57981
threads: 4
webport: 57991
zk: 'NO'
26 changes: 26 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57981/conf/route.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
backup:
- 127.0.0.1:57983
main:
- addr: 127.0.0.1:57980
buckets: &id001
- '0'
- '1'
- '2'
- '3'
- '4'
- '5'
- '6'
- '7'
- '8'
- '9'
- a
- b
- c
- d
- e
- f
- addr: 127.0.0.1:57981
buckets: *id001
- addr: 127.0.0.1:57982
buckets: *id001
numbucket: 16
32 changes: 32 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57982/conf/global.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
hstore:
data:
check_vhash: true
datafile_max_str: 4000M
flush_interval: 60
flush_wake_str: 10M
no_gc_days: 7
hint:
hint_index_interval_str: 32K
hint_merge_interval: 5
hint_no_merged: true
hint_split_cap_str: 1M
htree:
tree_height: 3
local:
home: /data
mc:
body_big_str: 5M
body_c_str: 0K
body_max_str: 50M
flush_max_str: 100M
max_key_len: 250
max_req: 16
server:
accesslog: /tmp/access.log
errorlog: /tmp/error.log
hostname: 127.0.0.1
listen: 0.0.0.0
port: 57982
threads: 4
webport: 57992
zk: 'NO'
26 changes: 26 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57982/conf/route.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
backup:
- 127.0.0.1:57983
main:
- addr: 127.0.0.1:57980
buckets: &id001
- '0'
- '1'
- '2'
- '3'
- '4'
- '5'
- '6'
- '7'
- '8'
- '9'
- a
- b
- c
- d
- e
- f
- addr: 127.0.0.1:57981
buckets: *id001
- addr: 127.0.0.1:57982
buckets: *id001
numbucket: 16
32 changes: 32 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57983/conf/global.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
hstore:
data:
check_vhash: true
datafile_max_str: 4000M
flush_interval: 60
flush_wake_str: 10M
no_gc_days: 7
hint:
hint_index_interval_str: 32K
hint_merge_interval: 5
hint_no_merged: true
hint_split_cap_str: 1M
htree:
tree_height: 3
local:
home: /data
mc:
body_big_str: 5M
body_c_str: 0K
body_max_str: 50M
flush_max_str: 100M
max_key_len: 250
max_req: 16
server:
accesslog: /tmp/access.log
errorlog: /tmp/error.log
hostname: 127.0.0.1
listen: 0.0.0.0
port: 57983
threads: 4
webport: 57993
zk: 'NO'
26 changes: 26 additions & 0 deletions .doubanpde/scripts/bdb/gobeansproxy/57983/conf/route.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
backup:
- 127.0.0.1:57983
main:
- addr: 127.0.0.1:57980
buckets: &id001
- '0'
- '1'
- '2'
- '3'
- '4'
- '5'
- '6'
- '7'
- '8'
- '9'
- a
- b
- c
- d
- e
- f
- addr: 127.0.0.1:57981
buckets: *id001
- addr: 127.0.0.1:57982
buckets: *id001
numbucket: 16
Loading
Loading