From c0a4a79616e68deedc6e414a4c7ac0510dacea3a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 17 Sep 2020 13:47:07 +0200 Subject: [PATCH 1/5] Add default config path to help --- graphite-ch-optimizer.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/graphite-ch-optimizer.go b/graphite-ch-optimizer.go index 713f749..10f8ac1 100644 --- a/graphite-ch-optimizer.go +++ b/graphite-ch-optimizer.go @@ -157,7 +157,8 @@ func setDefaultConfig() { func processFlags() error { // Parse command line arguments in differend flag groups pflag.CommandLine.SortFlags = false - customConfig := pflag.StringP("config", "c", "", "Filename of the custom config. CLI arguments override it") + defaultConfig := "/etc/" + filepath.Base(os.Args[0]) + "/config.toml" + customConfig := pflag.StringP("config", "c", defaultConfig, "Filename of the custom config. CLI arguments override it") pflag.Bool("print-defaults", false, "Print default config values and exit") pflag.BoolP("version", "v", false, "Print version and exit") @@ -215,10 +216,11 @@ func processFlags() error { // readConfigFile set file as the config name if it's not empty and reads the config from Viper.configPaths func readConfigFile(file string) error { var cfgNotFound viper.ConfigFileNotFoundError + var perr *os.PathError viper.SetConfigFile(file) err := viper.ReadInConfig() if err != nil { - if errors.As(err, &cfgNotFound) { + if errors.As(err, &cfgNotFound) || errors.As(err, &perr) { logrus.Debug("No config files were found, use defaults and flags") return nil } From 801ccc4c1be249dc289d2221185c916462c4243d Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 17 Sep 2020 15:33:12 +0200 Subject: [PATCH 2/5] Fix logging to file, set output for clickhouse debug logging --- graphite-ch-optimizer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graphite-ch-optimizer.go b/graphite-ch-optimizer.go index 10f8ac1..ceea663 100644 --- a/graphite-ch-optimizer.go +++ b/graphite-ch-optimizer.go @@ -116,12 +116,13 @@ func init() { case "-": output = os.Stdout default: - output, err = os.OpenFile(cfg.Logging.Output, os.O_WRONLY|os.O_APPEND, 0644) + output, err = os.OpenFile(cfg.Logging.Output, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { logrus.Fatalf("Unable to open file %s for writing: %v", cfg.Logging.Output, err) } } logrus.SetOutput(output) + clickhouse.SetLogOutput(output) level, err := logrus.ParseLevel(cfg.Logging.Level) if err != nil { logrus.Fatalf("Fail to parse log level: %v", err) From c7d37ab9c4b0b23e4bf50aace80c631e64c31730 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 1 Oct 2020 20:02:50 +0200 Subject: [PATCH 3/5] Add default for config path, fix the main query --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1c5a3e7..9032133 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,8 @@ INNER JOIN WHERE p.active AND ((toDateTime(p.max_date + 1) + g.age) < now()) GROUP BY table, - partition + partition, + partition_id HAVING (modified_at < rollup_time) OR (parts > 1) ORDER BY table ASC, @@ -175,7 +176,7 @@ Possible command line arguments: ``` Usage of graphite-ch-optimizer: - -c, --config string Filename of the custom config. CLI arguments override it + -c, --config string Filename of the custom config. CLI arguments override it (default "/etc/graphite-ch-optimizer/config.toml") --print-defaults Print default config values and exit -v, --version Print version and exit --optimize-interval duration The partition will be merged after having no writes for more than the given duration (default 72h0m0s) @@ -185,4 +186,6 @@ Usage of graphite-ch-optimizer: --one-shot Program will make only one optimization instead of working in the loop (true if dry-run) --log-level string Valid options are: panic, fatal, error, warn, warning, info, debug, trace --output string The logs file. '-' is accepted as STDOUT (default "-") + +Version: version-string ``` From a8465781abafeed7b54a48c2d1f88baeed2cd25b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 2 Oct 2020 12:58:43 +0200 Subject: [PATCH 4/5] Add CodeQL analyze --- .github/workflows/codeql-analysis.yml | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..e3f2dc8 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['go'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 + From 923adecdfa4a50ba771378cd3fcec04d8d3d268b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 2 Oct 2020 13:01:44 +0200 Subject: [PATCH 5/5] Add latest go version into go matrix --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c52f251..3c2fd01 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,6 +16,8 @@ jobs: go: - ^1.13 - ^1.14 + - ^1.15 + - ^1 steps: - name: Set up Go 1.x