Merge pull request #7 from innogames/mat_view

Fix optimization for materialized views

.github/workflows/tests.yml

@@ -0,0 +1,35 @@
+name: Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  tests:
+    name: Test code
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go:
+          - ^1.13
+          - ^1.14
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go }}
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Test
+      run: make test
+
+    - name: Build and run version
+      run: |
+        make VERSION=testing-version
+        ./graphite-ch-optimizer --version

.github/workflows/upload-assets.yml

@@ -0,0 +1,80 @@
+name: Upload Packages to new release
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    container:
+      image: innogames/graphite-ch-optimizer:builder
+
+    outputs:
+      deb: ${{ steps.build.outputs.deb }}
+      rpm: ${{ steps.build.outputs.rpm }}
+      sha256sum: ${{ steps.build.outputs.sha256sum }}
+      md5sum: ${{ steps.build.outputs.md5sum }}
+
+    steps:
+      - uses: actions/checkout@v2
+        name: Checkout
+        with:
+          # Otherwise there's a risk to not get latest tag
+          # We hope, that the current commit at
+          # least 50 commits close to the latest release
+          fetch-depth: 50
+      - name: Build packages
+        id: build
+        run: |
+          # Checkout action doesn't fetch tags
+          git fetch --tags
+          make -e CGO_ENABLED=0 packages
+          make github_artifact
+      - name: Upload rpm
+        id: upload-rpm
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET: ${{ steps.build.outputs.rpm }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: artifact/${{ env.ASSET }}
+          asset_name: ${{ env.ASSET }}
+          asset_content_type: application/octet-stream
+      - name: Upload sha256sum
+        id: upload-sha256sum
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET: ${{ steps.build.outputs.sha256sum }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: artifact/${{ env.ASSET }}
+          asset_name: ${{ env.ASSET }}
+          asset_content_type: application/octet-stream
+      - name: Upload md5sum
+        id: upload-md5sum
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET: ${{ steps.build.outputs.md5sum }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: artifact/${{ env.ASSET }}
+          asset_name: ${{ env.ASSET }}
+          asset_content_type: application/octet-stream
+      - name: Upload deb
+        id: upload-deb
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET: ${{ steps.build.outputs.deb }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: artifact/${{ env.ASSET }}
+          asset_name: ${{ env.ASSET }}
+          asset_content_type: application/octet-stream

.gitignore

@@ -1,2 +1,3 @@
 graphite-ch-optimizer
 build/
+artifact/

Jenkinsfile

@@ -1,84 +1,40 @@
 #!/usr/bin/groovy
 library 'adminsLib@master'
-import groovy.json.JsonOutput
 
 properties([
     parameters([
         string(defaultValue: '', description: 'deb-drop repository, see https://github.com/innogames/deb-drop/', name: 'REPO_NAME', trim: false),
-        booleanParam(defaultValue: false, description: 'Publish new packages and images or not. True for new releases, False by default.', name: 'PUBLISH'),
     ])
 ])
 
-def build_packages(docker_image) {
-    env.IMAGE = docker_image.imageName()
-    sh '''
-        #!/bin/bash
-        docker run --rm "${IMAGE}" | tar x --wildcards 'build/graphite-ch-optimizer?*'
-        '''
-}
-
-
 // Remove builds in presented status, default is ['ABORTED', 'NOT_BUILT']
 jobCommon.cleanNotFinishedBuilds()
 
-node('docker') {
+node() {
 ansiColor('xterm') {
     // Checkout repo and get info about current stage
     sh 'echo Initial env; env | sort'
     env.PACKAGE_NAME = 'graphite-ch-optimizer'
-    env.DOCKER_IMAGE = 'innogames/' + env.PACKAGE_NAME
-    def img_builder
-    def img_build
     try {
         stage('Checkout') {
             gitSteps checkout: true, changeBuildName: false
             sh 'set +x; echo "Environment variables after checkout:"; env|sort'
             env.NEW_VERSION = sh(returnStdout: true, script: 'make version').trim()
             currentBuild.displayName = "${currentBuild.number}: version ${env.NEW_VERSION}"
         }
-        stage('Tests') {
-            try {
-                docker.image("${env.DOCKER_IMAGE}:builder").pull()
-            } catch (all) {
-                echo 'Unable to pull builder image, building from scratch'
-            }
-            img_builder = docker.build(
-                    "${env.DOCKER_IMAGE}:builder",
-                    "--target builder --cache-from=${env.DOCKER_IMAGE}:builder ."
-                    )
-            // Make test is the part of the build image
-            img_build = docker.build("${env.DOCKER_IMAGE}:build", '--target build .')
-            if (env.GIT_BRANCH == 'master') {
-                docker.withRegistry('', 'dockerIgSysadminsToken') {
-                    img_builder.push()
+        stage('Upload to deb-drop') {
+            when(env.GIT_BRANCH_OR_TAG == 'tag' && jobCommon.launchedByUser() && env.REPO_NAME != '') {
+                deb_package = "graphite-ch-optimizer_${env.NEW_VERSION}_amd64.deb"
+                [deb_package, 'md5sum', 'sha256sum'].each { file->
+                    sh "set -ex; wget https://github.com/innogames/graphite-ch-optimizer/releases/download/${env.GIT_BRANCH}/${file}"
+                }
+                ['md5sum', 'sha256sum'].each { sum->
+                    sh "set -ex; ${sum} --ignore-missing --status -c ${sum}"
                 }
-            }
-        }
-        stage('Building') {
-        when(jobCommon.launchedByUser()) {
-            build_packages(img_build)
-
-            if (env.REPO_NAME) {
-                deb_packages = findFiles(glob: "build/*${env.NEW_VERSION}*deb")
                 withCredentials([string(credentialsId: 'DEB_DROP_TOKEN', variable: 'DebDropToken')]) {
-                    deb_packages.each { pack->
-                        jobCommon.uploadPackage  file: pack.path, repo: env.REPO_NAME, token: DebDropToken
-                    }
+                    jobCommon.uploadPackage  file: deb_package, repo: env.REPO_NAME, token: DebDropToken
                 }
             }
-
-        }
-        when(env.GIT_BRANCH_OR_TAG == 'tag' && !jobCommon.launchedByUser()) {
-            // TODO: implement github-api requests in library to publish new releases and assets
-            echo 'Assets publishing will be here eventually'
-            //build_packages(img_build)
-            //withCredentials([usernamePassword(
-            //    credentialsId: 'username/token',
-            //    usernameVariable: 'USERNAME',
-            //    passwordVariable: 'TOKEN'
-            //)]) {
-            //}
-        }
         }
         cleanWs(notFailBuild: true)
     }

Makefile

@@ -8,37 +8,62 @@ define DESC =
 endef
 GO_FILES = $(shell find -name '*.go')
 PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm
+SUM_FILES = build/sha256sum build/md5sum
 
-.PHONY: clean all version test
+
+.PHONY: all clean docker test version
 
 all: build
 
 version:
 	@echo $(VERSION)
 
 clean:
+	rm -rf artifact
 	rm -rf build
 	rm -rf $(NAME)
 
 rebuild: clean all
 
+# Run tests
 test:
 	go vet $(GO_FILES)
 	go test $(GO_FILES)
 
-build: $(NAME)
+build: | $(NAME)
+	mkdir build
+
+docker:
+	docker build -t innogames/$(NAME):builder -f docker/builder/Dockerfile .
+	docker build -t innogames/$(NAME):latest -f docker/$(NAME)/Dockerfile .
 
 $(NAME): $(NAME).go
 	go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
 
-build/$(NAME): $(NAME).go
-	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
+#########################################################
+# Prepare artifact directory and set outputs for upload #
+#########################################################
+github_artifact: $(foreach art,$(PKG_FILES) $(SUM_FILES), artifact/$(notdir $(art)))
 
-build/config.toml.example: build/$(NAME)
-	./build/$(NAME) --print-defaults > $@
+artifact:
+	mkdir $@
+
+# Link artifact to directory with setting step output to filename
+artifact/%: ART=$(notdir $@)
+artifact/%: TYPE=$(lastword $(subst ., ,$(ART)))
+artifact/%: build/% | artifact
+	cp -l $< $@
+	@echo '::set-output name=$(TYPE)::$(ART)'
+
+#######
+# END #
+#######
 
-packages: $(PKG_FILES)
+#############
+# Packaging #
+#############
 
+# Prepare everything for packaging
 .ONESHELL:
 build/pkg: build/$(NAME) build/config.toml.example
 	cd build
@@ -47,6 +72,22 @@ build/pkg: build/$(NAME) build/config.toml.example
 	cp -l $(NAME) pkg/usr/bin/
 	cp -l config.toml.example pkg/etc/$(NAME)
 
+build/$(NAME): $(NAME).go
+	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
+
+build/config.toml.example: build/$(NAME)
+	./build/$(NAME) --print-defaults > $@
+
+packages: $(PKG_FILES) $(SUM_FILES)
+
+# md5 and sha256 sum-files for packages
+$(SUM_FILES): COMMAND = $(notdir $@)
+$(SUM_FILES): PKG_FILES_NAME = $(notdir $(PKG_FILES))
+.ONESHELL:
+$(SUM_FILES): $(PKG_FILES)
+	cd build
+	$(COMMAND) $(PKG_FILES_NAME) > $(COMMAND)
+
 deb: $(word 1, $(PKG_FILES))
 
 rpm: $(word 2, $(PKG_FILES))
@@ -70,3 +111,7 @@ $(PKG_FILES): build/pkg
 		-p build \
 		build/pkg/=/ \
 		packaging/$(NAME).service=/lib/systemd/system/$(NAME).service
+
+#######
+# END #
+#######

README.md

@@ -14,6 +14,10 @@ make rpm
 ```
 
 ## Docker
+
+To build docker image locally run:  
+`make docker`
+
 To launch the container run the following command on the host with a running ClickHouse server:  
 `docker run --net=host --rm innogames/graphite-ch-optimizer:latest`
 
@@ -22,7 +26,7 @@ To launch the container run the following command on the host with a running Cli
 * Daemon mode is preferable over one-shot script for the normal work
 * It's safe to run it on the cluster hosts
 * You could either run it on the one of replicated host or just over the all hosts
-* If you have big partitions (month or something like this) and will get exceptions about timeout, then you need to adjust `read_timeout` parameter in DSN
+* If you have big partitions (month or something like this) and will get exceptions about timeout, then you need to adjust `receive_timeout` parameter in DSN
 * `optimize_throw_if_noop=1` is not mandatory, but good to have.
 * The next picture demonstrates the result of running the daemon for the first time on ~3 years old GraphiteMergeTree table:  
 <img src="./docs/result.jpg" alt="example"/>
@@ -148,7 +152,7 @@ Default config:
 ```toml
 [clickhouse]
   optimize-interval = "72h0m0s"
-  server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&read_timeout=3600&debug=true"
+  server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true"
 
 [daemon]
   dry-run = false
@@ -168,7 +172,7 @@ Usage of graphite-ch-optimizer:
       --print-defaults               Print default config values and exit
   -v, --version                      Print version and exit
       --optimize-interval duration   The active partitions won't be optimized more than once per this interval, seconds (default 72h0m0s)
-  -s, --server-dsn string            DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&read_timeout=3600&debug=true")
+  -s, --server-dsn string            DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true")
   -n, --dry-run                      Will print how many partitions would be merged without actions
       --loop-interval duration       Daemon will check if there partitions to merge once per this interval, seconds (default 1h0m0s)
       --one-shot                     Program will make only one optimization instead of working in the loop (true if dry-run)

docker/builder/Dockerfile

@@ -0,0 +1,8 @@
+#
+# Image for building packages in docker
+# innogames/graphite-ch-optimizer:builder on docker-hub
+#
+FROM golang:1.13-alpine as builder
+
+RUN apk --no-cache add ruby ruby-dev ruby-etc make gcc g++ rpm git tar && \
+    gem install --no-user-install --no-document fpm

Dockerfile → docker/graphite-ch-optimizer/Dockerfile

@@ -1,17 +1,7 @@
-#
-# Image for building packages in docker
-# innogames/graphite-ch-optimizer:builder on docker-hub
-#
-FROM golang:1.13-alpine as builder
-
-RUN apk --no-cache add ruby ruby-dev ruby-etc make gcc g++ rpm git tar && \
-    gem install --no-user-install --no-document fpm
-
-
 #
 # Image which contains the binary artefacts
 #
-FROM builder AS build
+FROM innogames/graphite-ch-optimizer:builder AS build
 COPY . ./graphite-ch-optimizer
 
 WORKDIR ./graphite-ch-optimizer

docs/DOCKER.md

@@ -0,0 +1,6 @@
+# Docker images and tags
+The images `innogames/graphite-ch-optimizer` are available on docker-hub with the following tags:
+
+- `${semantic_version}` (`docker/graphite-ch-optimizer/Dockerfile`) - these tags are automatically built from tags with tag regexp `v([.0-9]+)`
+- `latest` (`docker/graphite-ch-optimizer/Dockerfile`) - is built automatically from the latest `master` commit
+- `builder` (`docker/builder/Dockerfile`) - is built automatically from the latest `master` commit

docs/RELEASE.md

@@ -0,0 +1,6 @@
+# CREATE NEW RELEASE
+When the code is ready to new release, create it with the tag `v${major_version}.${minor_version}.${patch_version}`.  
+We have a [workflow](../.github/workflows/upload-assets.yml), which will upload created DEB and RPM packages together with hash sums as the release assets.
+
+## Use Jenkins to upload packages to deb-drop repository
+When the release is ready and assets are uploaded, launch the multibranch pipeline job configured against [Jenkinsfile](../Jenkinsfile) with desired version. It will download the package, compare hashsums and upload it to the repository.