Skip to content

Commit

Permalink
Merge pull request #16 from innogames/issue15
Browse files Browse the repository at this point in the history
Fix client timeout and some side improvements
  • Loading branch information
Felixoid authored Nov 17, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents f1e59f0 + e5d979e commit d2f7c42
Showing 695 changed files with 261,990 additions and 50 deletions.
12 changes: 8 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -9,7 +9,11 @@ endef
GO_FILES = $(shell find -name '*.go')
PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm
SUM_FILES = build/sha256sum build/md5sum
MODULE = github.com/innogames/$(NAME)

GO ?= go
export GOFLAGS += -mod=vendor
export GO111MODULE := on

.PHONY: all clean docker test version

@@ -27,8 +31,8 @@ rebuild: clean all

# Run tests
test:
go vet $(GO_FILES)
go test $(GO_FILES)
$(GO) vet $(MODULE)
$(GO) test $(MODULE)

build: | $(NAME)
mkdir build
@@ -38,7 +42,7 @@ docker:
docker build -t innogames/$(NAME):latest -f docker/$(NAME)/Dockerfile .

$(NAME): $(NAME).go
go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
$(GO) build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .

#########################################################
# Prepare artifact directory and set outputs for upload #
@@ -73,7 +77,7 @@ build/pkg: build/$(NAME) build/config.toml.example
cp -l config.toml.example pkg/etc/$(NAME)

build/$(NAME): $(NAME).go
GOOS=linux GOARCH=amd64 go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
GOOS=linux GOARCH=amd64 $(GO) build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .

build/config.toml.example: build/$(NAME)
./build/$(NAME) --print-defaults > $@
8 changes: 5 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -33,7 +33,9 @@ To launch the container run the following command on the host with a running Cli
* Daemon mode is preferable over one-shot script for the normal work
* It's safe to run it on the cluster hosts
* You could either run it on the one of replicated host or just over the all hosts
* If you have big partitions (month or something like this) and will get exceptions about timeout, then you need to adjust `receive_timeout` parameter in DSN
* There are two different parameters in DSN, that should be adjusted together to fix timeouts for big partitions (month or something like this) optimizing:
* `read_timeout` - `clickhouse-go` parameter. Controls the timeout between the `graphite-ch-optimizer` and ClickHouse server.
* `receive_timeout` - ClickHouse parameter, used when the OPTIMIZE query is applied in the cluster. See [comment](https://github.com/ClickHouse/ClickHouse/issues/4831#issuecomment-708721042) in the issue.
* `optimize_throw_if_noop=1` is not mandatory, but good to have.
* The next picture demonstrates the result of running the daemon for the first time on ~3 years old GraphiteMergeTree table:
<img src="./docs/result.jpg" alt="example"/>
@@ -160,7 +162,7 @@ Default config:
```toml
[clickhouse]
optimize-interval = "72h0m0s"
server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true"
server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600"

[daemon]
dry-run = false
@@ -180,7 +182,7 @@ Usage of graphite-ch-optimizer:
--print-defaults Print default config values and exit
-v, --version Print version and exit
--optimize-interval duration The partition will be merged after having no writes for more than the given duration (default 72h0m0s)
-s, --server-dsn string DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true")
-s, --server-dsn string DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600")
-n, --dry-run Will print how many partitions would be merged without actions
--loop-interval duration Daemon will check if there partitions to merge once per this interval (default 1h0m0s)
--one-shot Program will make only one optimization instead of working in the loop (true if dry-run)
21 changes: 11 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
@@ -3,17 +3,18 @@ module github.com/innogames/graphite-ch-optimizer
go 1.13

require (
github.com/ClickHouse/clickhouse-go v1.3.13
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/pelletier/go-toml v1.6.0
github.com/sirupsen/logrus v1.4.2
github.com/spf13/afero v1.2.2 // indirect
github.com/ClickHouse/clickhouse-go v1.4.3
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/magiconair/properties v1.8.4 // indirect
github.com/mitchellh/mapstructure v1.3.3 // indirect
github.com/pelletier/go-toml v1.8.1
github.com/sirupsen/logrus v1.7.0
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.6.2
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 // indirect
golang.org/x/text v0.3.2 // indirect
gopkg.in/ini.v1 v1.51.1 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
github.com/spf13/viper v1.7.1
golang.org/x/sys v0.0.0-20201027140754-0fcbb8f4928c // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
)
Loading

0 comments on commit d2f7c42

Please sign in to comment.